html2rss 0.32.4 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -2
  3. data/lib/html2rss/article/deduplicator.rb +5 -2
  4. data/lib/html2rss/article/enclosure.rb +185 -25
  5. data/lib/html2rss/article/title.rb +203 -0
  6. data/lib/html2rss/article.rb +79 -13
  7. data/lib/html2rss/auto_source/README.md +16 -8
  8. data/lib/html2rss/auto_source/cleanup.rb +127 -53
  9. data/lib/html2rss/auto_source/corroboration.rb +167 -0
  10. data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
  11. data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
  12. data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
  13. data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
  14. data/lib/html2rss/auto_source/page_links.rb +164 -0
  15. data/lib/html2rss/auto_source/scraper/html.rb +11 -45
  16. data/lib/html2rss/auto_source/scraper/json_document.rb +3 -6
  17. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +202 -125
  18. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +27 -38
  19. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
  20. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +27 -35
  21. data/lib/html2rss/auto_source/scraper/json_state.rb +4 -9
  22. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +27 -5
  23. data/lib/html2rss/auto_source/scraper/microdata.rb +233 -248
  24. data/lib/html2rss/auto_source/scraper/microformats2.rb +10 -4
  25. data/lib/html2rss/auto_source/scraper/native_feed.rb +28 -18
  26. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
  27. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
  28. data/lib/html2rss/auto_source/scraper/schema/thing.rb +140 -22
  29. data/lib/html2rss/auto_source/scraper/schema.rb +52 -8
  30. data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
  31. data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -20
  32. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
  33. data/lib/html2rss/auto_source/scraper/sitemap.rb +7 -14
  34. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
  35. data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
  36. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +7 -7
  37. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +2 -4
  38. data/lib/html2rss/auto_source/scraper.rb +10 -13
  39. data/lib/html2rss/auto_source/segment.rb +19 -5
  40. data/lib/html2rss/auto_source/segmenter/cluster.rb +18 -22
  41. data/lib/html2rss/auto_source/segmenter/list.rb +35 -59
  42. data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
  43. data/lib/html2rss/auto_source/segmenter/semantic.rb +23 -19
  44. data/lib/html2rss/auto_source/segmenter.rb +4 -8
  45. data/lib/html2rss/auto_source.rb +81 -26
  46. data/lib/html2rss/batch.rb +24 -32
  47. data/lib/html2rss/capture/README.md +6 -3
  48. data/lib/html2rss/capture.rb +51 -59
  49. data/lib/html2rss/channel.rb +30 -12
  50. data/lib/html2rss/cli/render.rb +97 -96
  51. data/lib/html2rss/cli/validate.rb +2 -4
  52. data/lib/html2rss/cli.rb +55 -38
  53. data/lib/html2rss/config/auto_source_contract.rb +52 -45
  54. data/lib/html2rss/config/issue_mapper.rb +4 -1
  55. data/lib/html2rss/config/request_controls.rb +16 -5
  56. data/lib/html2rss/config/schema.rb +23 -15
  57. data/lib/html2rss/config/selectors_validator.rb +3 -2
  58. data/lib/html2rss/config/validation_issue.rb +4 -1
  59. data/lib/html2rss/config/validator.rb +14 -3
  60. data/lib/html2rss/config.rb +54 -28
  61. data/lib/html2rss/contract_error.rb +9 -0
  62. data/lib/html2rss/defaults.rb +25 -38
  63. data/lib/html2rss/doctor/botasaurus.rb +18 -22
  64. data/lib/html2rss/error.rb +17 -3
  65. data/lib/html2rss/error_kind.rb +39 -0
  66. data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
  67. data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
  68. data/lib/html2rss/feed_builder/json_feed.rb +2 -2
  69. data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
  70. data/lib/html2rss/feed_builder/rss.rb +50 -7
  71. data/lib/html2rss/feed_builder.rb +2 -1
  72. data/lib/html2rss/feed_pipeline/README.md +1 -1
  73. data/lib/html2rss/feed_pipeline/auto_fallback.rb +16 -34
  74. data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
  75. data/lib/html2rss/feed_pipeline.rb +40 -27
  76. data/lib/html2rss/feed_resolution/README.md +1 -1
  77. data/lib/html2rss/feed_resolution/candidate_generator.rb +6 -8
  78. data/lib/html2rss/feed_resolution/options.rb +3 -7
  79. data/lib/html2rss/feed_resolution/policy.rb +29 -33
  80. data/lib/html2rss/feed_resolution/probe.rb +6 -4
  81. data/lib/html2rss/feed_resolution/scorer.rb +29 -29
  82. data/lib/html2rss/feed_resolution.rb +16 -21
  83. data/lib/html2rss/feed_result.rb +11 -6
  84. data/lib/html2rss/hash_util.rb +91 -90
  85. data/lib/html2rss/html/article_extractor/category_extractor.rb +7 -10
  86. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  87. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -33
  88. data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
  89. data/lib/html2rss/html/article_extractor/id_generator.rb +7 -10
  90. data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
  91. data/lib/html2rss/html/article_extractor.rb +19 -47
  92. data/lib/html2rss/html/article_rules/category.rb +3 -3
  93. data/lib/html2rss/html/article_rules/description.rb +6 -15
  94. data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
  95. data/lib/html2rss/html/article_rules/image.rb +30 -16
  96. data/lib/html2rss/html/feed_link.rb +7 -9
  97. data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
  98. data/lib/html2rss/html/navigator.rb +39 -9
  99. data/lib/html2rss/html/probe.rb +55 -51
  100. data/lib/html2rss/html/rendering/description_builder.rb +46 -32
  101. data/lib/html2rss/html/rendering.rb +2 -9
  102. data/lib/html2rss/html/sst_article_extractor.rb +55 -145
  103. data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
  104. data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
  105. data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
  106. data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
  107. data/lib/html2rss/html/title_pieces.rb +169 -0
  108. data/lib/html2rss/html/visibility.rb +56 -0
  109. data/lib/html2rss/html.rb +2 -0
  110. data/lib/html2rss/link_destination/destination_facts.rb +17 -1
  111. data/lib/html2rss/link_destination/noise_policy.rb +11 -10
  112. data/lib/html2rss/link_destination/path_classifier.rb +111 -42
  113. data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
  114. data/lib/html2rss/link_destination/sponsorship.rb +184 -0
  115. data/lib/html2rss/link_destination/text_classifier.rb +104 -8
  116. data/lib/html2rss/mcp/README.md +13 -1
  117. data/lib/html2rss/mcp/config_argument.rb +14 -4
  118. data/lib/html2rss/mcp/contract.rb +20 -9
  119. data/lib/html2rss/mcp/outcome.rb +18 -17
  120. data/lib/html2rss/mcp/runtime.rb +53 -23
  121. data/lib/html2rss/mcp/server/tools.rb +9 -9
  122. data/lib/html2rss/mcp/server.rb +13 -4
  123. data/lib/html2rss/option.rb +48 -0
  124. data/lib/html2rss/page_recon/README.md +1 -1
  125. data/lib/html2rss/page_recon/diagnostics.rb +9 -9
  126. data/lib/html2rss/page_recon.rb +0 -4
  127. data/lib/html2rss/recon.rb +19 -23
  128. data/lib/html2rss/request_service/blocked_surface.rb +5 -8
  129. data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
  130. data/lib/html2rss/request_service/botasaurus_strategy.rb +44 -12
  131. data/lib/html2rss/request_service/budget.rb +3 -24
  132. data/lib/html2rss/request_service/compressed_body.rb +36 -48
  133. data/lib/html2rss/request_service/context.rb +16 -17
  134. data/lib/html2rss/request_service/cookie_jar.rb +89 -0
  135. data/lib/html2rss/request_service/cookie_store.rb +79 -0
  136. data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
  137. data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
  138. data/lib/html2rss/request_service/local_file_strategy.rb +2 -3
  139. data/lib/html2rss/request_service/network_guard.rb +3 -2
  140. data/lib/html2rss/request_service/policy.rb +55 -36
  141. data/lib/html2rss/request_service/replay_breaker.rb +56 -0
  142. data/lib/html2rss/request_service/response.rb +33 -19
  143. data/lib/html2rss/request_service/response_cache.rb +109 -0
  144. data/lib/html2rss/request_service/response_caching.rb +95 -0
  145. data/lib/html2rss/request_service/strategy.rb +5 -11
  146. data/lib/html2rss/request_service.rb +71 -12
  147. data/lib/html2rss/request_session/pager/base.rb +3 -10
  148. data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
  149. data/lib/html2rss/request_session/pager/offset.rb +1 -7
  150. data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
  151. data/lib/html2rss/request_session/pager/url_template.rb +1 -7
  152. data/lib/html2rss/request_session/pager.rb +1 -1
  153. data/lib/html2rss/request_session.rb +6 -2
  154. data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
  155. data/lib/html2rss/scoring/container_assessor.rb +12 -13
  156. data/lib/html2rss/scoring/engine.rb +10 -38
  157. data/lib/html2rss/scoring/link_resolver.rb +2 -7
  158. data/lib/html2rss/scoring/observation.rb +2 -2
  159. data/lib/html2rss/selector_candidates.rb +19 -8
  160. data/lib/html2rss/selectors/README.md +1 -1
  161. data/lib/html2rss/selectors/extractors/href.rb +2 -4
  162. data/lib/html2rss/selectors/json_xml.rb +33 -34
  163. data/lib/html2rss/selectors/option_spec.rb +4 -1
  164. data/lib/html2rss/selectors/post_processors/base.rb +0 -8
  165. data/lib/html2rss/selectors/post_processors/gsub.rb +9 -10
  166. data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
  167. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
  168. data/lib/html2rss/selectors/post_processors/parse_time.rb +5 -6
  169. data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
  170. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
  171. data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
  172. data/lib/html2rss/selectors/schema_export.rb +60 -60
  173. data/lib/html2rss/selectors.rb +67 -30
  174. data/lib/html2rss/sst/attrs.rb +2 -2
  175. data/lib/html2rss/sst/document.rb +3 -3
  176. data/lib/html2rss/sst/index.rb +30 -11
  177. data/lib/html2rss/sst/node.rb +42 -29
  178. data/lib/html2rss/sst/normalizer.rb +47 -34
  179. data/lib/html2rss/sst/tags.rb +113 -2
  180. data/lib/html2rss/sst/text.rb +94 -58
  181. data/lib/html2rss/sst.rb +2 -0
  182. data/lib/html2rss/status.rb +27 -31
  183. data/lib/html2rss/surface_category.rb +1 -1
  184. data/lib/html2rss/syndication/README.md +12 -6
  185. data/lib/html2rss/syndication/alternate_links.rb +39 -0
  186. data/lib/html2rss/syndication/channel_facts.rb +87 -0
  187. data/lib/html2rss/syndication/discovery.rb +171 -187
  188. data/lib/html2rss/syndication/feed.rb +20 -0
  189. data/lib/html2rss/syndication/item_media.rb +103 -0
  190. data/lib/html2rss/syndication/parser.rb +117 -116
  191. data/lib/html2rss/test/enhance_audit.rb +15 -17
  192. data/lib/html2rss/test/item_sample.rb +43 -0
  193. data/lib/html2rss/test/policy.rb +9 -11
  194. data/lib/html2rss/test.rb +66 -57
  195. data/lib/html2rss/url.rb +163 -62
  196. data/lib/html2rss/version.rb +1 -1
  197. data/lib/html2rss/word_count.rb +53 -0
  198. data/lib/html2rss.rb +19 -9
  199. data/schema/html2rss-config.schema.json +56 -41
  200. metadata +33 -4
  201. data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
  202. data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
@@ -11,6 +11,9 @@ module Html2rss
11
11
  # introspection SoT (+for+ / +expectation_for+); {SchemaExport} maps Ruby
12
12
  # types to JSON Schema only.
13
13
  class OptionSpec
14
+ # Raised when an extractor/post-processor +OPTIONS+ declares a type with no JSON Schema mapping.
15
+ class InvalidError < Html2rss::ContractError; end
16
+
14
17
  # Maps Ruby option types to JSON Schema +type+ strings.
15
18
  RUBY_TO_JSON_TYPE = {
16
19
  String => 'string',
@@ -83,7 +86,7 @@ module Html2rss
83
86
  when Array then ruby_type.map { |t| json_type_for(t) }
84
87
  else
85
88
  RUBY_TO_JSON_TYPE.fetch(ruby_type) do
86
- raise ArgumentError, "unsupported OPTIONS type mapping for #{ruby_type}"
89
+ raise InvalidError, "unsupported OPTIONS type mapping for #{ruby_type}"
87
90
  end
88
91
  end
89
92
  end
@@ -114,14 +114,6 @@ module Html2rss
114
114
  end
115
115
 
116
116
  attr_reader :value, :context
117
-
118
- # Abstract method to be implemented by subclasses
119
- #
120
- # @return [Object] transformed value
121
- # @raise [NotImplementedError] if not implemented in subclass
122
- def call
123
- raise NotImplementedError, 'You must implement the `call` method in the post processor'
124
- end
125
117
  end
126
118
  end
127
119
  end
@@ -70,7 +70,7 @@ module Html2rss
70
70
  #
71
71
  # @param string [String]
72
72
  # @return [Regexp]
73
- # @raise [ArgumentError] when +string+ is not a String, exceeds the character cap,
73
+ # @raise [InvalidOption] when +string+ is not a String, exceeds the character cap,
74
74
  # contains nested quantifiers, or does not parse
75
75
  def self.compiled_pattern(string)
76
76
  compiled_patterns[string] ||= compile_regexp_string(string)
@@ -88,25 +88,25 @@ module Html2rss
88
88
  # to maintain backwards compatibility before building the Regexp.
89
89
  # Length is the character count of the authored string (the JSON Schema
90
90
  # +maxLength+ unit), then the AST is walked for nested quantifiers,
91
- # then +to_re+. Parser errors are re-raised as ArgumentError so
91
+ # then +to_re+. Parser errors are re-raised as InvalidOption so
92
92
  # validation and execution share one path.
93
93
  #
94
94
  # @param string [String]
95
95
  # @return [Regexp]
96
- # @raise [ArgumentError] when +string+ is not a String, exceeds {MAX_PATTERN_LENGTH},
96
+ # @raise [InvalidOption] when +string+ is not a String, exceeds {MAX_PATTERN_LENGTH},
97
97
  # contains nested quantifiers, or does not parse
98
98
  def self.compile_regexp_string(string)
99
- raise ArgumentError, 'must be a string!' unless string.is_a?(String)
100
- raise ArgumentError, "pattern exceeds #{MAX_PATTERN_LENGTH} characters" if string.length > MAX_PATTERN_LENGTH
99
+ raise InvalidOption, 'must be a string!' unless string.is_a?(String)
100
+ raise InvalidOption, "pattern exceeds #{MAX_PATTERN_LENGTH} characters" if string.length > MAX_PATTERN_LENGTH
101
101
 
102
102
  source = regexp_source(string)
103
103
 
104
- expression = Regexp::Parser.parse(source, options: ::Regexp::EXTENDED | ::Regexp::IGNORECASE)
105
- raise ArgumentError, 'pattern contains nested quantifiers' if nested_quantifiers?(expression)
104
+ expression = Regexp::Parser.parse(source)
105
+ raise InvalidOption, 'pattern contains nested quantifiers' if nested_quantifiers?(expression)
106
106
 
107
107
  expression.to_re
108
108
  rescue Regexp::Parser::Error => error
109
- raise ArgumentError, error.message, [], cause: nil
109
+ raise InvalidOption, error.message, [], cause: nil
110
110
  end
111
111
  private_class_method :compile_regexp_string
112
112
 
@@ -161,8 +161,7 @@ module Html2rss
161
161
  ##
162
162
  # @return [Regexp]
163
163
  def pattern
164
- raw = @pattern
165
- raw.is_a?(String) ? self.class.compiled_pattern(raw) : raw
164
+ self.class.compiled_pattern(@pattern)
166
165
  end
167
166
  end
168
167
  end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ class Selectors
5
+ module PostProcessors
6
+ module HtmlTransformers
7
+ ##
8
+ # Transformer that keeps only media a reader can play: an +http(s)+ +src+ on +<audio>+, +<video>+
9
+ # and +<source>+, made absolute; any other +src+ (missing, blank, +javascript:+, +data:+, unparsable)
10
+ # is removed. A +<source>+ left without one is dropped, and so is an +<audio>+ or +<video>+ with no
11
+ # playable +src+ of its own or on a child +<source>+ (no empty player shell).
12
+ class KeepPlayableMedia
13
+ # Elements that play a file named by +src+ (theirs or a child +<source>+).
14
+ PLAYERS = %w[audio video].freeze
15
+
16
+ # @param base_url [String, Html2rss::Url] base URL used to resolve relative sources
17
+ def initialize(base_url)
18
+ @base_url = base_url
19
+ end
20
+
21
+ ##
22
+ # @param node_name [String] node name currently being transformed
23
+ # @param node [Nokogiri::XML::Node] node currently being transformed
24
+ # @param _env [Hash] transformer context
25
+ # @option _env [Object] :_reserved reserved for transformer pipeline context
26
+ # @return [void]
27
+ def call(node_name:, node:, **_env)
28
+ if node_name == 'source'
29
+ node.unlink unless absolutize_src(node)
30
+ elsif PLAYERS.include?(node_name)
31
+ # Children are transformed after their parent, so child sources are only checked here.
32
+ playable = absolutize_src(node) || node.css('source').any? { Url.resolve_web(_1['src'], @base_url) }
33
+ node.unlink unless playable
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ # Rewrites +src+ to its absolute playable URL (Nokogiri stores its +to_s+) or removes it;
40
+ # returns the URL kept, if any.
41
+ def absolutize_src(node)
42
+ url = Url.resolve_web(node['src'], @base_url)
43
+ url ? node['src'] = url : node.remove_attribute('src')
44
+ url
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -12,9 +12,7 @@ module Html2rss
12
12
  URL_ELEMENTS_WITH_URL_ATTRIBUTE = {
13
13
  'a' => :href, # Visible link
14
14
  'img' => :src, # Visible image
15
- 'iframe' => :src, # Embedded frame (visible content)
16
- 'audio' => :src, # Can show controls, so potentially visible
17
- 'video' => :src # Video player is visible
15
+ 'iframe' => :src # Embedded frame (visible content)
18
16
  }.freeze
19
17
 
20
18
  # @param base_url [String, Html2rss::Url] base URL used to resolve relative links
@@ -63,15 +63,14 @@ module Html2rss
63
63
  # @param _value [String] extracted time string (unused; type-checked via VALUE_TYPE)
64
64
  # @param context [Selectors::StepEnv] must carry a usable +time_zone+
65
65
  # @return [void]
66
- # @raise [ArgumentError] when time_zone is nil or empty
66
+ # @raise [MissingOption] when time_zone is nil or empty
67
67
  def self.validate_args!(_value, context)
68
68
  time_zone_value = context.time_zone
69
69
 
70
- if time_zone_value.nil? || time_zone_value.empty?
71
- raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil
72
- end
70
+ raise MissingOption, 'time_zone cannot be nil or empty', [], cause: nil if time_zone_value.nil?
73
71
 
74
72
  assert_type(time_zone_value, String, :time_zone, context:)
73
+ raise MissingOption, 'time_zone cannot be nil or empty', [], cause: nil if time_zone_value.empty?
75
74
  end
76
75
 
77
76
  ##
@@ -89,7 +88,7 @@ module Html2rss
89
88
  private
90
89
 
91
90
  def parse_time(string, time_zone)
92
- components = Date._parse(string, false)
91
+ components = Date._parse(string)
93
92
  if components[:zone] || components.key?(:offset)
94
93
  Time.parse(string)
95
94
  else
@@ -104,7 +103,7 @@ module Html2rss
104
103
  components[:mday],
105
104
  components[:hour] || 0,
106
105
  components[:min] || 0,
107
- (components[:sec] || 0) + (components[:sec_fraction] || 0)
106
+ components[:sec] || 0
108
107
  )
109
108
  end
110
109
  end
@@ -44,7 +44,7 @@ module Html2rss
44
44
  # @param _context [Selectors::StepEnv] post-processor context
45
45
  # @return [void]
46
46
  def self.validate_args!(value, _context)
47
- raise ArgumentError, 'The `value` option is missing or empty.' if value.to_s.empty?
47
+ raise MissingOption, 'The `value` option is missing or empty.' if value.to_s.empty?
48
48
  end
49
49
 
50
50
  ##
@@ -17,12 +17,14 @@ module Html2rss
17
17
  # - `referrer-policy='no-referrer'` to <img> tags
18
18
  # - wraps all <img> tags, whose direct parent is not an <a>, into an <a>
19
19
  # linking to the <img>'s `src`.
20
+ # - keeps <audio>, <video> and <source> with an absolute http(s) `src`; a player
21
+ # without a playable `src` (its own or a <source>'s) is removed.
20
22
  #
21
23
  # Imagine this HTML structure:
22
24
  #
23
25
  # <section>
24
26
  # Lorem <b>ipsum</b> dolor...
25
- # <iframe src="https://evil.corp/miner"></iframe>
27
+ # <iframe src="https://evil.example/miner"></iframe>
26
28
  # <script>alert();</script>
27
29
  # </section>
28
30
  #
@@ -93,10 +95,20 @@ module Html2rss
93
95
  'audio' => {
94
96
  'referrerpolicy' => 'no-referrer',
95
97
  'crossorigin' => 'anonymous',
96
- 'preload' => 'none'
98
+ 'preload' => 'none',
99
+ 'controls' => 'true'
97
100
  }
98
101
  }.freeze
99
102
 
103
+ # Media elements kept in descriptions and the attributes they keep besides the html2rss defaults
104
+ # ({TAG_ATTRIBUTES}); {HtmlTransformers::KeepPlayableMedia} keeps +src+ only as an absolute
105
+ # +http(s)+ URL and drops a player left without one.
106
+ MEDIA_ATTRIBUTES = {
107
+ 'audio' => %w[src],
108
+ 'video' => %w[src],
109
+ 'source' => %w[src type]
110
+ }.freeze
111
+
100
112
  # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
101
113
  def self.schema_export = SchemaExport.for_post_processor(name: :sanitize_html, klass: self)
102
114
 
@@ -145,24 +157,25 @@ module Html2rss
145
157
  ##
146
158
  # @param base_url [String, Html2rss::Url]
147
159
  # @return [Hash] the memoized sanitize configuration
148
- # rubocop:disable-next Metrics/MethodLength, ThreadSafety/ClassInstanceVariable
160
+ # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, ThreadSafety/ClassInstanceVariable
149
161
  def self.sanitize_config(base_url)
150
162
  @sanitize_configs ||= {}
151
163
  @sanitize_configs[base_url] ||= begin
152
164
  config = Sanitize::Config.merge(
153
165
  Sanitize::Config::RELAXED,
154
- attributes: { all: %w[dir lang alt title translate] },
166
+ attributes: { all: %w[dir lang alt title translate], **MEDIA_ATTRIBUTES },
155
167
  add_attributes: TAG_ATTRIBUTES,
156
168
  transformers: [
157
169
  lambda { |env|
158
170
  HtmlTransformers::TransformUrlsToAbsoluteOnes.new(base_url).call(**env)
159
171
  },
172
+ ->(env) { HtmlTransformers::KeepPlayableMedia.new(base_url).call(**env) },
160
173
  ->(env) { HtmlTransformers::WrapImgInA.new.call(**env) }
161
174
  ]
162
175
  )
163
176
  config[:elements].delete('style')
164
177
  config.delete(:css)
165
- config[:elements].push('audio', 'video', 'source')
178
+ config[:elements].push(*MEDIA_ATTRIBUTES.keys)
166
179
  config.freeze
167
180
  end
168
181
  end
@@ -70,7 +70,7 @@ module Html2rss
70
70
  return (start..) unless step_config.key?(:end)
71
71
 
72
72
  finish = step_config[:end]
73
- raise ArgumentError, 'The `start` value must be unequal to the `end` value.' if start == finish
73
+ raise InvalidOption, 'The `start` value must be unequal to the `end` value.' if start == finish
74
74
 
75
75
  (start..finish)
76
76
  end
@@ -8,72 +8,72 @@ module Html2rss
8
8
  # Each registry class owns +DESCRIPTION+, +EXAMPLES+, and optional +OPTIONS+;
9
9
  # this module is the export adapter only.
10
10
  module SchemaExport
11
- module_function
12
-
13
- ##
14
- # @param name [Symbol, String] registry key
15
- # @param klass [Class] post-processor class
16
- # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
17
- def for_post_processor(name:, klass:)
18
- name = name.to_s
19
- {
20
- type: 'object',
21
- title: name,
22
- description: klass::DESCRIPTION,
23
- examples: klass::EXAMPLES,
24
- properties: post_processor_properties(name, klass),
25
- required: post_processor_required(klass),
26
- additionalProperties: true
27
- }
28
- end
11
+ class << self
12
+ ##
13
+ # @param name [Symbol, String] registry key
14
+ # @param klass [Class] post-processor class
15
+ # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
16
+ def for_post_processor(name:, klass:)
17
+ name = name.to_s
18
+ {
19
+ type: 'object',
20
+ title: name,
21
+ description: klass::DESCRIPTION,
22
+ examples: klass::EXAMPLES,
23
+ properties: post_processor_properties(name, klass),
24
+ required: post_processor_required(klass),
25
+ additionalProperties: true
26
+ }
27
+ end
29
28
 
30
- ##
31
- # @param name [Symbol, String] registry key
32
- # @param klass [Class] extractor class
33
- # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
34
- def for_extractor(name:, klass:)
35
- name = name.to_s
36
- {
37
- type: 'string',
38
- const: name,
39
- title: name,
40
- description: klass::DESCRIPTION,
41
- examples: klass::EXAMPLES
42
- }
43
- end
29
+ ##
30
+ # @param name [Symbol, String] registry key
31
+ # @param klass [Class] extractor class
32
+ # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
33
+ def for_extractor(name:, klass:)
34
+ name = name.to_s
35
+ {
36
+ type: 'string',
37
+ const: name,
38
+ title: name,
39
+ description: klass::DESCRIPTION,
40
+ examples: klass::EXAMPLES
41
+ }
42
+ end
44
43
 
45
- ##
46
- # @param name [String] registry key
47
- # @param klass [Class]
48
- # @return [Hash{Symbol => Hash}]
49
- def post_processor_properties(name, klass)
50
- properties = { name: { type: 'string', const: name } }
51
- OptionSpec.for(klass).each do |spec|
52
- properties[spec.name] = property_schema_for(spec.type)
44
+ ##
45
+ # @param name [String] registry key
46
+ # @param klass [Class]
47
+ # @return [Hash{Symbol => Hash}]
48
+ def post_processor_properties(name, klass)
49
+ properties = { name: { type: 'string', const: name } }
50
+ OptionSpec.for(klass).each do |spec|
51
+ properties[spec.name] = property_schema_for(spec.type)
52
+ end
53
+ properties
53
54
  end
54
- properties
55
- end
56
55
 
57
- ##
58
- # @param klass [Class]
59
- # @return [Array<String>]
60
- def post_processor_required(klass)
61
- required = ['name']
62
- required + OptionSpec.for(klass).select(&:required).map { |spec| spec.name.to_s }
63
- end
56
+ ##
57
+ # @param klass [Class]
58
+ # @return [Array<String>]
59
+ def post_processor_required(klass)
60
+ required = ['name']
61
+ required + OptionSpec.for(klass).select(&:required).map { |spec| spec.name.to_s }
62
+ end
64
63
 
65
- ##
66
- # @param ruby_type [Class, Array<Class>]
67
- # @return [Hash{Symbol => String, Array<String>}]
68
- def property_schema_for(ruby_type)
69
- { type: json_type_for(ruby_type) }
70
- end
64
+ ##
65
+ # @param ruby_type [Class, Array<Class>]
66
+ # @return [Hash{Symbol => String, Array<String>}]
67
+ def property_schema_for(ruby_type)
68
+ { type: json_type_for(ruby_type) }
69
+ end
71
70
 
72
- ##
73
- # @param ruby_type [Class, Array<Class>]
74
- # @return [String, Array<String>]
75
- def json_type_for(ruby_type)
76
- OptionSpec.json_type_for(ruby_type)
71
+ ##
72
+ # @param ruby_type [Class, Array<Class>]
73
+ # @return [String, Array<String>]
74
+ def json_type_for(ruby_type)
75
+ OptionSpec.json_type_for(ruby_type)
76
+ end
77
77
  end
78
78
  end
79
79
  end
@@ -35,6 +35,10 @@ module Html2rss
35
35
  # Config selector keys that map onto a different {Article} attribute.
36
36
  # +:enclosure+ stays singular in YAML; Article stores +:enclosures+.
37
37
  SELECTOR_TO_ARTICLE_KEY = { enclosure: :enclosures }.freeze
38
+ # Admission-drop reason tallied for selector items that carry no URL (same shape as
39
+ # {AutoSource#admission_drops}).
40
+ MISSING_URL_REASON = 'missing_url'
41
+
38
42
  # Selector keys that may be copied onto an Article (PROVIDED_KEYS + mapped aliases).
39
43
  SELECTABLE_SELECTOR_KEYS = Set[*(Html2rss::Article::PROVIDED_KEYS + SELECTOR_TO_ARTICLE_KEY.keys)].freeze
40
44
  # ArticleExtractor keys that enhance can fill. Skip the extractor when all are already present.
@@ -56,6 +60,7 @@ module Html2rss
56
60
  @time_zone = time_zone
57
61
  @rss_item_attributes = @selectors.keys.select { SELECTABLE_SELECTOR_KEYS.include?(_1) }
58
62
  @parsed_body = nil
63
+ @unlinked_count = 0
59
64
  @compiled_fields = compile_selector_fields
60
65
  @compiled_item_fields = @rss_item_attributes.filter_map { @compiled_fields[_1] }
61
66
  end
@@ -70,39 +75,52 @@ module Html2rss
70
75
  end
71
76
 
72
77
  ##
73
- # Iterates over each scraped article.
78
+ # Drop tallies for the last iteration: items skipped for having no URL, as {AutoSource#admission_drops}.
79
+ #
80
+ # @return [Hash{String => Integer}] empty when every item was linked
81
+ def admission_drops
82
+ @unlinked_count.positive? ? { MISSING_URL_REASON => @unlinked_count }.freeze : {}.freeze
83
+ end
84
+
85
+ ##
86
+ # Iterates over each scraped article. An item without a URL is skipped and tallied in {#admission_drops},
87
+ # as in auto-source ({Html2rss::Article#linked?}).
74
88
  #
75
89
  # @yield [article] Gives each article as an Html2rss::Article.
76
90
  # @return [Enumerator] An enumerator if no block is given.
77
91
  def each(&)
78
- return enum_for(:each) unless block_given?
92
+ return enum_for unless block_given?
79
93
 
80
94
  enhance = enhance?
95
+ @unlinked_count = 0
81
96
 
82
97
  parsed_body.css(items_selector).each do |item|
83
98
  article_hash = extract_article(item)
84
99
 
85
100
  enhance_article_hash(article_hash, item) if enhance
86
101
 
87
- yield Html2rss::Article.new(**article_hash, scraper: self.class)
102
+ article = build_article(article_hash)
103
+ article.linked? ? yield(article) : @unlinked_count += 1
88
104
  end
89
105
  end
90
106
 
91
107
  ##
92
108
  # Yields +baseline+, +enhanced+, and the item node per list entry for enhance diagnostics.
109
+ #
110
+ # @param linked_only [Boolean] skip items {#each} would drop for having no URL
93
111
  # Consumed by {Test::EnhanceAudit} so the extraction loop has one owner.
94
112
  #
95
113
  # @yieldparam baseline [Hash]
96
114
  # @yieldparam enhanced [Hash]
97
115
  # @yieldparam item [Nokogiri::XML::Element]
98
116
  # @return [Enumerator] if no block is given
99
- def each_enhance_pair
100
- return enum_for(:each_enhance_pair) unless block_given?
117
+ def each_enhance_pair(linked_only: false)
118
+ return enum_for(:each_enhance_pair, linked_only:) unless block_given?
101
119
 
102
120
  parsed_body.css(items_selector).each do |item|
103
121
  baseline = extract_article(item)
104
122
  enhanced = enhance_article_hash(baseline.dup, item)
105
- yield baseline, enhanced, item
123
+ yield baseline, enhanced, item unless linked_only && !build_article(enhanced).linked?
106
124
  end
107
125
  end
108
126
 
@@ -132,7 +150,7 @@ module Html2rss
132
150
 
133
151
  field = @compiled_fields[name]
134
152
  return select_compiled(field, scope) if field
135
- return select_field(name, scope:, config: nil) if name == :url
153
+ return select_field(name, scope:) if name == :url
136
154
 
137
155
  raise InvalidSelectorName, "Selector for '#{name}' is not defined."
138
156
  end
@@ -168,6 +186,12 @@ module Html2rss
168
186
  text.empty? || text == DEFAULT_ITEMS_SELECTOR
169
187
  end
170
188
 
189
+ ##
190
+ # Whether +selectors.items.enhance+ is on: each matched item node fills fields that explicit
191
+ # selectors left empty (see {ENHANCEABLE_KEYS}) via {Html::ArticleExtractor}, using only the
192
+ # item's own markup and making no extra requests.
193
+ #
194
+ # @return [Boolean]
171
195
  def enhance? = !!@selectors.dig(ITEMS_SELECTOR_KEY, :enhance)
172
196
 
173
197
  def parsed_body
@@ -183,6 +207,8 @@ module Html2rss
183
207
  end
184
208
  end
185
209
 
210
+ def build_article(article_hash) = Article.new(**article_hash, scraper: self.class)
211
+
186
212
  def extract_article(item)
187
213
  scope = item_env_for(item, @url)
188
214
  hash = @compiled_item_fields.each_with_object({}) do |field, h|
@@ -190,13 +216,30 @@ module Html2rss
190
216
  next if value.nil?
191
217
 
192
218
  article_key = field.article_key
193
- h[article_key] = article_key == :enclosures ? wrap_enclosure_value(value) : value
219
+ h[article_key] = article_value(article_key, value, scope)
194
220
  end
195
221
 
196
222
  hash[:url] ||= default_item_url(item) if anchor_element?(item)
197
223
  hash
198
224
  end
199
225
 
226
+ def article_value(article_key, value, scope)
227
+ case article_key
228
+ when :enclosures then wrap_enclosure_value(value)
229
+ when :url then resolve_url_value(value, scope.base_url)
230
+ else value
231
+ end
232
+ end
233
+
234
+ # A +url+ selector value that is a single whitespace-free string token (a +static+ value or a post-processor
235
+ # result) resolves against the page URL like an +href+; a value that is no http(s) reference stays as given.
236
+ # Text with surrounding words is left to +Article#url+'s sanitizer, which extracts the embedded URL.
237
+ def resolve_url_value(value, base_url)
238
+ return value unless value.is_a?(String) && value.match?(/\A\S+\z/)
239
+
240
+ Url.resolve_web(value, base_url) || value
241
+ end
242
+
200
243
  def enhance_article_hash(article_hash, article_tag) # rubocop:disable Metrics/MethodLength -- merge extracted keys
201
244
  return article_hash if enhanceable_fields_present?(article_hash)
202
245
 
@@ -205,7 +248,6 @@ module Html2rss
205
248
  article_tag,
206
249
  base_url: @url,
207
250
  selected_anchor:,
208
- fallback_anchorless: true,
209
251
  time_zone: @time_zone
210
252
  )
211
253
  return article_hash unless extracted
@@ -225,15 +267,11 @@ module Html2rss
225
267
  value.is_a?(Array) ? value : [value]
226
268
  end
227
269
 
228
- def select_field(selector_key, scope:, config:)
229
- return default_item_url(scope.item, scope.base_url) if fallback_url_selector?(selector_key, config, scope.item)
230
- raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined." if config.nil?
270
+ # Only reached for +:url+ without a configured selector, so the item must be an anchor.
271
+ def select_field(selector_key, scope:)
272
+ return default_item_url(scope.item, scope.base_url) if anchor_element?(scope.item)
231
273
 
232
- if SPECIAL_ATTRIBUTES.member?(selector_key)
233
- select_special(selector_key, scope:, config:)
234
- else
235
- select_regular(selector_key, scope:, config:)
236
- end
274
+ raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined."
237
275
  end
238
276
 
239
277
  def select_compiled(field, scope)
@@ -327,6 +365,7 @@ module Html2rss
327
365
  end
328
366
 
329
367
  def post_process(scope, value, post_process_steps)
368
+ value = step_input(value)
330
369
  post_process_steps.each do |step_config|
331
370
  value = PostProcessors.call(step_config[:name], value, scope.context_for(step_config:))
332
371
  end
@@ -334,9 +373,15 @@ module Html2rss
334
373
  value
335
374
  end
336
375
 
376
+ # Post-processors work on text, so a URL value (from the href extractor) enters them as a String.
377
+ def step_input(value)
378
+ value.is_a?(Url) ? value.to_s : value
379
+ end
380
+
337
381
  def apply_compiled_post_process(scope, value, steps)
338
382
  return value unless value && steps
339
383
 
384
+ value = step_input(value)
340
385
  steps.each do |klass, step_config|
341
386
  value = klass.new(value, scope.context_for(step_config:)).call
342
387
  end
@@ -375,15 +420,8 @@ module Html2rss
375
420
  end
376
421
 
377
422
  def extract_category_text(category)
378
- text = case category
379
- when Nokogiri::XML::Node, Nokogiri::XML::NodeSet
380
- Html2rss::Html::Navigator.extract_visible_text(category)
381
- else
382
- category&.to_s
383
- end
384
-
385
- stripped = text&.strip
386
- stripped unless stripped.nil? || stripped.empty?
423
+ stripped = category.to_s.strip
424
+ stripped unless stripped.empty?
387
425
  end
388
426
 
389
427
  def node_set_with_multiple_elements?(nodes)
@@ -396,12 +434,11 @@ module Html2rss
396
434
  Extractors.element(item, config[:selector])
397
435
  end
398
436
 
399
- # @return [Hash, nil] enclosure details, or nil when the selector yields nothing.
437
+ # @return [Hash, nil] enclosure details, or nil when the selector yields nothing or no http(s) URL
438
+ # ({Url.resolve_web})
400
439
  def enclosure(scope:, config:)
401
440
  selected = select_regular(:enclosure, scope:, config:)
402
- return if selected.nil? || selected.to_s.strip.empty?
403
-
404
- url = Url.from_relative(selected, scope.base_url)
441
+ return unless (url = Url.resolve_web(selected, scope.base_url))
405
442
 
406
443
  { url:, type: config[:content_type] }
407
444
  end
@@ -65,7 +65,7 @@ module Html2rss
65
65
  case class_names
66
66
  when nil then AttrDefaults::EMPTY_CLASS_NAMES
67
67
  when Array then class_names.map { |token| token.to_s.freeze }.freeze
68
- else raise ArgumentError, "class_names must be an Array, got #{class_names.class}"
68
+ else raise InvalidError, "class_names must be an Array, got #{class_names.class}"
69
69
  end
70
70
  end
71
71
 
@@ -73,7 +73,7 @@ module Html2rss
73
73
  case raw
74
74
  when nil then AttrDefaults::EMPTY_RAW
75
75
  when Hash then raw.transform_values { |v| v.to_s.freeze }.freeze
76
- else raise ArgumentError, "raw must be a Hash, got #{raw.class}"
76
+ else raise InvalidError, "raw must be a Hash, got #{raw.class}"
77
77
  end
78
78
  end
79
79
  end
@@ -13,10 +13,10 @@ module Html2rss
13
13
  # @param degraded [Boolean] true when MAX_NODES forced semantic-tag-only mode
14
14
  # @param node_count [Integer]
15
15
  # @return [Document]
16
- # @raise [ArgumentError] when root/index missing
16
+ # @raise [InvalidError] when root/index missing
17
17
  def self.build(root:, index:, degraded: false, node_count: 0)
18
- raise ArgumentError, 'root is required' unless root.is_a?(Node)
19
- raise ArgumentError, 'index is required' unless index.is_a?(Index)
18
+ raise InvalidError, 'root is required' unless root.is_a?(Node)
19
+ raise InvalidError, 'index is required' unless index.is_a?(Index)
20
20
 
21
21
  new(root:, index:, degraded: !!degraded, node_count: Integer(node_count))
22
22
  end