html2rss 0.22.2 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +11 -108
  3. data/lib/html2rss/{articles → article}/deduplicator.rb +3 -4
  4. data/lib/html2rss/{rss_builder → article}/enclosure.rb +6 -22
  5. data/lib/html2rss/article.rb +213 -0
  6. data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb +80 -0
  7. data/lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb +86 -0
  8. data/lib/html2rss/auto_source/discovery/dom_clustering.rb +119 -0
  9. data/lib/html2rss/auto_source/discovery/list_candidates.rb +94 -0
  10. data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb +219 -0
  11. data/lib/html2rss/auto_source/discovery/semantic_containers.rb +71 -0
  12. data/lib/html2rss/auto_source/discovery/sitemap.rb +159 -0
  13. data/lib/html2rss/auto_source/discovery.rb +14 -0
  14. data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb +28 -0
  15. data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb +106 -0
  16. data/lib/html2rss/auto_source/link_heuristics/container_signals.rb +80 -0
  17. data/lib/html2rss/auto_source/link_heuristics/destination_facts.rb +42 -0
  18. data/lib/html2rss/auto_source/link_heuristics/href_extractor.rb +38 -0
  19. data/lib/html2rss/auto_source/link_heuristics/path_classifier.rb +221 -0
  20. data/lib/html2rss/auto_source/link_heuristics/text_classifier.rb +66 -0
  21. data/lib/html2rss/auto_source/link_heuristics.rb +139 -0
  22. data/lib/html2rss/auto_source/scraper/html.rb +22 -33
  23. data/lib/html2rss/auto_source/scraper/json_state.rb +1 -1
  24. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +163 -0
  25. data/lib/html2rss/auto_source/scraper/microformats2.rb +146 -0
  26. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +29 -114
  27. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +50 -11
  28. data/lib/html2rss/auto_source/scraper/schema/thing.rb +35 -7
  29. data/lib/html2rss/auto_source/scraper/schema.rb +61 -18
  30. data/lib/html2rss/auto_source/scraper/semantic_html/{deduplicator.rb → entry_deduplicator.rb} +3 -3
  31. data/lib/html2rss/auto_source/scraper/semantic_html.rb +23 -171
  32. data/lib/html2rss/auto_source/scraper/sitemap.rb +153 -0
  33. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb +110 -0
  34. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -78
  35. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +9 -0
  36. data/lib/html2rss/auto_source/scraper.rb +10 -3
  37. data/lib/html2rss/auto_source.rb +30 -35
  38. data/lib/html2rss/channel.rb +198 -0
  39. data/lib/html2rss/cli.rb +5 -6
  40. data/lib/html2rss/config/auto_source_contract.rb +51 -0
  41. data/lib/html2rss/config/request_controls.rb +136 -0
  42. data/lib/html2rss/config/request_headers.rb +1 -1
  43. data/lib/html2rss/config/schema.rb +65 -114
  44. data/lib/html2rss/{selectors/config.rb → config/selectors_validator.rb} +56 -7
  45. data/lib/html2rss/config/validator.rb +6 -6
  46. data/lib/html2rss/config.rb +176 -19
  47. data/lib/html2rss/{configuration.rb → defaults.rb} +12 -10
  48. data/lib/html2rss/feed_builder/item_presentation.rb +42 -0
  49. data/lib/html2rss/feed_builder/json_feed/item.rb +112 -0
  50. data/lib/html2rss/feed_builder/json_feed.rb +81 -0
  51. data/lib/html2rss/feed_builder/rss/stylesheet.rb +65 -0
  52. data/lib/html2rss/feed_builder/rss.rb +116 -0
  53. data/lib/html2rss/feed_builder.rb +15 -0
  54. data/lib/html2rss/feed_pipeline/auto_fallback.rb +120 -60
  55. data/lib/html2rss/feed_pipeline/runtime_policy.rb +108 -0
  56. data/lib/html2rss/feed_pipeline/strategy_plan.rb +64 -0
  57. data/lib/html2rss/feed_pipeline.rb +83 -89
  58. data/lib/html2rss/feed_result.rb +92 -0
  59. data/lib/html2rss/html/article_extractor/category_extractor.rb +123 -0
  60. data/lib/html2rss/html/article_extractor/date_extractor.rb +22 -0
  61. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +95 -0
  62. data/lib/html2rss/html/article_extractor/heading_extractor.rb +54 -0
  63. data/lib/html2rss/html/article_extractor/id_generator.rb +69 -0
  64. data/lib/html2rss/html/article_extractor/image_extractor.rb +60 -0
  65. data/lib/html2rss/html/article_extractor.rb +134 -0
  66. data/lib/html2rss/html/navigator/text_extractor.rb +79 -0
  67. data/lib/html2rss/html/navigator.rb +146 -0
  68. data/lib/html2rss/html/rendering/audio_renderer.rb +38 -0
  69. data/lib/html2rss/html/rendering/description_builder.rb +89 -0
  70. data/lib/html2rss/html/rendering/image_renderer.rb +43 -0
  71. data/lib/html2rss/html/rendering/media_renderer.rb +39 -0
  72. data/lib/html2rss/html/rendering/pdf_renderer.rb +36 -0
  73. data/lib/html2rss/html/rendering/video_renderer.rb +38 -0
  74. data/lib/html2rss/html/rendering.rb +25 -0
  75. data/lib/html2rss/html.rb +8 -0
  76. data/lib/html2rss/request_service/blocked_surface.rb +67 -0
  77. data/lib/html2rss/request_service/botasaurus_contract.rb +33 -9
  78. data/lib/html2rss/request_service/botasaurus_strategy.rb +21 -30
  79. data/lib/html2rss/request_service/browserless_strategy.rb +2 -23
  80. data/lib/html2rss/request_service/budget.rb +97 -18
  81. data/lib/html2rss/request_service/context.rb +1 -3
  82. data/lib/html2rss/request_service/faraday_strategy.rb +12 -26
  83. data/lib/html2rss/request_service/local_file_strategy.rb +12 -4
  84. data/lib/html2rss/request_service/network_guard.rb +125 -0
  85. data/lib/html2rss/request_service/policy.rb +10 -82
  86. data/lib/html2rss/request_service/puppet_commander/navigation_guards.rb +148 -0
  87. data/lib/html2rss/request_service/puppet_commander/preload_runner.rb +86 -0
  88. data/lib/html2rss/request_service/puppet_commander.rb +22 -191
  89. data/lib/html2rss/request_service/response.rb +9 -1
  90. data/lib/html2rss/request_service/response_guard.rb +1 -1
  91. data/lib/html2rss/request_service/strategy.rb +123 -5
  92. data/lib/html2rss/request_service.rb +7 -2
  93. data/lib/html2rss/request_session/pager/base.rb +83 -0
  94. data/lib/html2rss/request_session/pager/custom_selector.rb +39 -0
  95. data/lib/html2rss/request_session/pager/json_cursor.rb +78 -0
  96. data/lib/html2rss/request_session/pager/offset.rb +48 -0
  97. data/lib/html2rss/request_session/pager/rel_next.rb +23 -0
  98. data/lib/html2rss/request_session/pager/url_template.rb +48 -0
  99. data/lib/html2rss/request_session/pager.rb +75 -0
  100. data/lib/html2rss/request_session.rb +29 -16
  101. data/lib/html2rss/selectors/item_scope.rb +30 -0
  102. data/lib/html2rss/selectors/post_processors/base.rb +1 -1
  103. data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +0 -1
  104. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +0 -2
  105. data/lib/html2rss/selectors/post_processors/template.rb +7 -3
  106. data/lib/html2rss/selectors.rb +84 -64
  107. data/lib/html2rss/status.rb +132 -0
  108. data/lib/html2rss/version.rb +1 -3
  109. data/lib/html2rss.rb +34 -22
  110. data/schema/html2rss-config.schema.json +109 -9
  111. metadata +71 -43
  112. data/lib/html2rss/auto_source/scraper/html/class_clustering.rb +0 -196
  113. data/lib/html2rss/auto_source/scraper/link_heuristics.rb +0 -401
  114. data/lib/html2rss/auto_source/scraper/schema/list_item.rb +0 -28
  115. data/lib/html2rss/auto_source/scraper/semantic_html/anchor_selector.rb +0 -49
  116. data/lib/html2rss/blocked_surface.rb +0 -65
  117. data/lib/html2rss/category_extractor.rb +0 -116
  118. data/lib/html2rss/config/class_methods.rb +0 -183
  119. data/lib/html2rss/html_extractor/date_extractor.rb +0 -20
  120. data/lib/html2rss/html_extractor/enclosure_extractor.rb +0 -91
  121. data/lib/html2rss/html_extractor/heading_extractor.rb +0 -50
  122. data/lib/html2rss/html_extractor/id_generator.rb +0 -67
  123. data/lib/html2rss/html_extractor/image_extractor.rb +0 -58
  124. data/lib/html2rss/html_extractor/list_candidates.rb +0 -88
  125. data/lib/html2rss/html_extractor/semantic_anchor_candidates.rb +0 -260
  126. data/lib/html2rss/html_extractor/semantic_containers.rb +0 -69
  127. data/lib/html2rss/html_extractor/text_extractor.rb +0 -77
  128. data/lib/html2rss/html_extractor.rb +0 -190
  129. data/lib/html2rss/html_navigator.rb +0 -71
  130. data/lib/html2rss/json_feed_builder/item.rb +0 -94
  131. data/lib/html2rss/json_feed_builder.rb +0 -59
  132. data/lib/html2rss/rendering/audio_renderer.rb +0 -36
  133. data/lib/html2rss/rendering/description_builder.rb +0 -87
  134. data/lib/html2rss/rendering/image_renderer.rb +0 -41
  135. data/lib/html2rss/rendering/media_renderer.rb +0 -37
  136. data/lib/html2rss/rendering/pdf_renderer.rb +0 -34
  137. data/lib/html2rss/rendering/video_renderer.rb +0 -36
  138. data/lib/html2rss/rendering.rb +0 -23
  139. data/lib/html2rss/request_controls.rb +0 -133
  140. data/lib/html2rss/request_session/rel_next_pager.rb +0 -70
  141. data/lib/html2rss/request_session/runtime_input.rb +0 -71
  142. data/lib/html2rss/request_session/runtime_policy.rb +0 -84
  143. data/lib/html2rss/rss_builder/article.rb +0 -208
  144. data/lib/html2rss/rss_builder/channel.rb +0 -118
  145. data/lib/html2rss/rss_builder/stylesheet.rb +0 -63
  146. data/lib/html2rss/rss_builder.rb +0 -101
@@ -3,7 +3,7 @@
3
3
  module Html2rss
4
4
  class Config
5
5
  ##
6
- # Builds the exported configuration JSON Schema from the runtime validators.
6
+ # JSON export adapter from the runtime validators to the packaged schema artifact.
7
7
  module Schema
8
8
  module_function
9
9
 
@@ -60,7 +60,8 @@ module Html2rss
60
60
  def call
61
61
  schema = validator_schema
62
62
  apply_top_level(schema)
63
- assign_properties(schema.fetch(:properties))
63
+ schema.fetch(:properties).merge!(overlay)
64
+ schema.fetch(:properties).delete(:dynamic_params_error)
64
65
  DeepStringifier.call(schema)
65
66
  end
66
67
 
@@ -78,124 +79,74 @@ module Html2rss
78
79
  ]
79
80
  end
80
81
 
81
- def assign_properties(properties)
82
- properties.merge!(
83
- strategy: Components.strategy,
84
- headers: Components.headers,
85
- stylesheets: Components.stylesheets,
86
- auto_source: Components.auto_source,
87
- selectors: Components.selectors
88
- )
89
- properties.delete(:dynamic_params_error)
90
- end
91
- end
92
-
93
- ##
94
- # Exposes schema fragments that populate the top-level configuration schema.
95
- module Components
96
- module_function
97
-
98
- # @return [Hash{Symbol => Object}] schema fragment for strategy selection
99
- def strategy
100
- {
101
- type: 'string',
102
- not: { type: 'null' }
103
- }
104
- end
105
-
106
- # @return [Hash{Symbol => Object}] schema fragment for headers
107
- def headers
108
- {
109
- type: 'object',
110
- description: 'HTTP headers applied to every request.',
111
- additionalProperties: { type: 'string' }
112
- }
113
- end
114
-
115
- # @return [Hash{Symbol => Object}] schema fragment for stylesheet definitions
116
- def stylesheets
117
- {
118
- type: 'array',
119
- description: 'Collection of stylesheets to attach to the RSS feed.',
120
- items: Html2rss::Config::Validator::StylesheetConfig.json_schema(loose: true)
121
- }
122
- end
123
-
124
- # @return [Hash{Symbol => Object}] schema fragment for auto_source configuration
125
- def auto_source
126
- schema = Html2rss::AutoSource::Config.json_schema(loose: true)
127
- schema[:default] = DeepStringifier.call(Html2rss::AutoSource::DEFAULT_CONFIG)
128
- schema
129
- end
130
-
131
- # @return [Hash{Symbol => Object}] schema fragment for selectors configuration
132
- def selectors
133
- Selectors.schema
134
- end
135
- end
136
-
137
- ##
138
- # Provides schema fragments that document selector configuration.
139
- module Selectors
140
- module_function
141
-
142
- # Pattern used for dynamic selector keys excluding reserved selector names.
143
- RESERVED_SELECTOR_PATTERN = '^(?!items$|enclosure$|guid$|categories$).+$'
144
-
145
- # @return [Hash{Symbol => Object}] schema fragment for selectors root object
146
- def schema
147
- {
148
- type: 'object',
149
- description: 'Selectors used to extract article attributes.',
150
- properties: selector_properties,
151
- patternProperties: pattern_properties,
152
- additionalProperties: true
82
+ # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
83
+ def overlay
84
+ items_schema = Html2rss::Config::SelectorsValidator::Items.new.schema.json_schema(loose: true)
85
+ items_schema[:properties][:pagination] = {
86
+ description: 'Pagination configuration or maximum page count integer.',
87
+ oneOf: [
88
+ { type: 'integer', exclusiveMinimum: 0 },
89
+ {
90
+ type: 'object',
91
+ properties: {
92
+ strategy: { type: 'string', enum: Html2rss::RequestSession::Pager.strategy_names },
93
+ max_pages: { type: 'integer', exclusiveMinimum: 0 },
94
+ selector: { type: 'string' },
95
+ param: { type: 'string' },
96
+ start_page: { type: 'integer' },
97
+ step: { type: 'integer', exclusiveMinimum: 0 },
98
+ start_offset: { type: 'integer' },
99
+ increment: { type: 'integer', exclusiveMinimum: 0 },
100
+ cursor_path: { type: 'string' },
101
+ next_url_path: { type: 'string' }
102
+ },
103
+ additionalProperties: true
104
+ }
105
+ ]
153
106
  }
154
- end
155
107
 
156
- # rubocop:disable Layout/LineLength
157
- # @return [Hash{Symbol => Object}] schema map for reserved selector properties
158
- def selector_properties
159
108
  {
160
- items: items_schema,
161
- enclosure: enclosure_schema,
162
- guid: reference_array('List of selector keys used to build the GUID. Each entry must reference a sibling selector key; runtime validation enforces those references.'),
163
- categories: reference_array('List of selector keys whose values will be used as categories. Each entry must reference a sibling selector key; runtime validation enforces those references.')
109
+ strategy: {
110
+ type: 'string',
111
+ not: { type: 'null' }
112
+ },
113
+ headers: {
114
+ type: 'object',
115
+ description: 'HTTP headers applied to every request.',
116
+ additionalProperties: { type: 'string' }
117
+ },
118
+ stylesheets: {
119
+ type: 'array',
120
+ description: 'Collection of stylesheets to attach to the RSS feed.',
121
+ items: Html2rss::Config::Validator::StylesheetConfig.json_schema(loose: true)
122
+ },
123
+ auto_source: Html2rss::Config::AutoSourceContract.json_schema(loose: true).merge(
124
+ default: DeepStringifier.call(Html2rss::AutoSource::DEFAULT_CONFIG)
125
+ ),
126
+ selectors: {
127
+ type: 'object',
128
+ description: 'Selectors used to extract article attributes.',
129
+ properties: {
130
+ items: items_schema.merge(
131
+ description: 'Defines the items selector and optional enhancement settings.'
132
+ ),
133
+ enclosure: Html2rss::Config::SelectorsValidator::Enclosure.new.schema.json_schema(loose: true).merge(
134
+ description: 'Describes enclosure extraction settings.'
135
+ ),
136
+ guid: reference_array('List of selector keys used to build the GUID. Each entry must reference a sibling selector key; runtime validation enforces those references.'),
137
+ categories: reference_array('List of selector keys whose values will be used as categories. Each entry must reference a sibling selector key; runtime validation enforces those references.')
138
+ },
139
+ patternProperties: {
140
+ '^(?!items$|enclosure$|guid$|categories$).+$' => Html2rss::Config::SelectorsValidator::Selector.new.schema.json_schema(loose: true).merge(
141
+ description: 'Dynamic selector definition keyed by attribute name.'
142
+ )
143
+ },
144
+ additionalProperties: true
145
+ }
164
146
  }
165
147
  end
166
- # rubocop:enable Layout/LineLength
167
-
168
- # @return [Hash{String => Object}] schema map for dynamic selector keys
169
- def pattern_properties
170
- { RESERVED_SELECTOR_PATTERN => dynamic_selector_schema }
171
- end
172
-
173
- # @return [Hash{Symbol => Object}] schema fragment for dynamic selector entries
174
- def dynamic_selector_schema
175
- Html2rss::Selectors::Config::Selector.new.schema.json_schema(loose: true).merge(
176
- description: 'Dynamic selector definition keyed by attribute name.'
177
- )
178
- end
179
-
180
- # @return [Hash{Symbol => Object}] schema fragment for `items` selector configuration
181
- def items_schema
182
- Html2rss::Selectors::Config::Items.new.schema.json_schema(loose: true).merge(
183
- description: 'Defines the items selector and optional enhancement settings.'
184
- )
185
- end
186
-
187
- # @return [Hash{Symbol => Object}] schema fragment for `enclosure` selector configuration
188
- def enclosure_schema
189
- Html2rss::Selectors::Config::Enclosure.new.schema.json_schema(loose: true).merge(
190
- description: 'Describes enclosure extraction settings.'
191
- )
192
- end
148
+ # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
193
149
 
194
- # JSON Schema can enforce non-empty reference arrays, while runtime
195
- # validation remains authoritative for checking that each entry points
196
- # to an existing sibling selector key.
197
- # @param description [String] human-readable description for the reference field
198
- # @return [Hash{Symbol => Object}] JSON schema fragment for selector references
199
150
  def reference_array(description)
200
151
  {
201
152
  type: 'array',
@@ -3,23 +3,72 @@
3
3
  require 'dry-validation'
4
4
 
5
5
  module Html2rss
6
- class Selectors
6
+ class Config
7
7
  ##
8
8
  # Validates the configuration hash for :selectors.
9
- class Config < Dry::Validation::Contract
9
+ class SelectorsValidator < Dry::Validation::Contract
10
10
  # Required wrapper key used to validate dynamic selector names.
11
11
  NESTING_KEY = :dynamic_keys_workaround
12
12
 
13
13
  ##
14
14
  # Validates the configuration of the :items selector
15
15
  class Items < Dry::Validation::Contract
16
+ # Supported pagination strategy names (sole source: RequestSession::Pager).
17
+ # @return [Array<String>]
18
+ STRATEGY_NAMES = Html2rss::RequestSession::Pager.strategy_names
19
+
16
20
  params do
17
21
  required(:selector).filled(:string)
18
22
  optional(:order).filled(included_in?: %w[reverse])
19
23
  optional(:enhance).filled(:bool?)
20
- optional(:pagination).hash do
21
- required(:max_pages).filled(:integer, gt?: 0)
24
+ optional(:pagination)
25
+ end
26
+
27
+ rule(:pagination) do
28
+ next if value.nil?
29
+
30
+ if value.is_a?(Integer)
31
+ key.failure('integer must be greater than 0') if value <= 0
32
+ elsif value.is_a?(Hash)
33
+ val_hash = value.transform_keys(&:to_sym)
34
+ validate_pagination_hash(val_hash, key)
35
+ else
36
+ key.failure('must be an integer or a hash')
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def validate_pagination_hash(val_hash, key)
43
+ # Fail loud on explicit nil/non-positive: truthy checks would silently accept max_pages: nil.
44
+ if val_hash.key?(:max_pages) && (!val_hash[:max_pages].is_a?(Integer) || val_hash[:max_pages] <= 0)
45
+ key.failure('`max_pages` must be an integer greater than 0')
46
+ end
47
+
48
+ strategy = (val_hash[:strategy] || 'rel_next').to_s
49
+ unless STRATEGY_NAMES.include?(strategy)
50
+ key.failure("`strategy` must be one of: #{STRATEGY_NAMES.join(', ')}")
22
51
  end
52
+
53
+ validate_strategy_fields(strategy, val_hash, key)
54
+ end
55
+
56
+ def validate_strategy_fields(strategy, val_hash, key)
57
+ validate_custom_selector(val_hash, key) if strategy == 'custom_selector'
58
+ validate_json_cursor(val_hash, key) if strategy == 'json_cursor'
59
+ end
60
+
61
+ def validate_custom_selector(val_hash, key)
62
+ return unless val_hash[:selector].nil? || val_hash[:selector].to_s.strip.empty?
63
+
64
+ key.failure('`custom_selector` strategy requires `selector` to be specified')
65
+ end
66
+
67
+ def validate_json_cursor(val_hash, key)
68
+ return unless (val_hash[:cursor_path].nil? || val_hash[:cursor_path].to_s.strip.empty?) &&
69
+ (val_hash[:next_url_path].nil? || val_hash[:next_url_path].to_s.strip.empty?)
70
+
71
+ key.failure('`json_cursor` strategy requires either `cursor_path` or `next_url_path`')
23
72
  end
24
73
  end
25
74
 
@@ -84,9 +133,9 @@ module Html2rss
84
133
  value.each_pair do |selector_key, selector|
85
134
  case selector_key.to_sym
86
135
  when Selectors::ITEMS_SELECTOR_KEY
87
- Items.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
136
+ Items.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
88
137
  when :enclosure
89
- Enclosure.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
138
+ Enclosure.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
90
139
  when :guid, :categories
91
140
  unless selector.is_a?(Array)
92
141
  key(selector_key).failure("`#{selector_key}` must be an array")
@@ -102,7 +151,7 @@ module Html2rss
102
151
  end
103
152
  else
104
153
  # From here on, the selector is found under its "dynamic" selector_key
105
- Selector.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
154
+ Selector.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
106
155
  end
107
156
  end
108
157
  end
@@ -10,11 +10,11 @@ module Html2rss
10
10
  # URI format used for channel URL validation.
11
11
  URI_REGEXP = Url::URI_REGEXP
12
12
  # Allowed stylesheet MIME types.
13
- STYLESHEET_TYPES = RssBuilder::Stylesheet::TYPES
13
+ STYLESHEET_TYPES = FeedBuilder::Rss::Stylesheet::TYPES
14
14
  # Optional language/region format (`en` or `en-US`).
15
15
  LANGUAGE_FORMAT_REGEX = /\A[a-z]{2}(-[A-Z]{2})?\z/
16
- # Baseline strategy enum exported in static schema artifacts.
17
- BASE_STRATEGY_OPTIONS = ([:auto] + Html2rss::RequestService.strategy_names.map(&:to_sym)).uniq.freeze
16
+ # Baseline strategy-plan enum (:auto plus concrete RequestService strategies).
17
+ BASE_STRATEGY_OPTIONS = Html2rss::FeedPipeline::StrategyPlan.accepted_names.freeze
18
18
 
19
19
  # Contract for the top-level `channel` section.
20
20
  ChannelConfig = Dry::Schema.Params do
@@ -91,7 +91,7 @@ module Html2rss
91
91
  required(:channel).hash(ChannelConfig)
92
92
  optional(:headers).hash
93
93
  optional(:stylesheets).array(StylesheetConfig)
94
- optional(:auto_source).hash(AutoSource::Config)
94
+ optional(:auto_source).hash(Config::AutoSourceContract)
95
95
  optional(:selectors).hash
96
96
  optional(:dynamic_params_error).maybe(:string)
97
97
  optional(:request).hash(RequestConfig)
@@ -111,7 +111,7 @@ module Html2rss
111
111
 
112
112
  rule(:strategy) do
113
113
  next if value.nil?
114
- next if value == :auto || Html2rss::RequestService.strategy_registered?(value)
114
+ next if Html2rss::FeedPipeline::StrategyPlan.valid?(value)
115
115
 
116
116
  key.failure("must be one of: #{BASE_STRATEGY_OPTIONS.join(', ')}")
117
117
  end
@@ -126,7 +126,7 @@ module Html2rss
126
126
  rule(:selectors) do
127
127
  next unless value
128
128
 
129
- errors = Html2rss::Selectors::Config.call(value).errors
129
+ errors = Config::SelectorsValidator.call(value).errors
130
130
  errors.each { |error| key(:selectors).failure(error.text) } unless errors.empty?
131
131
  end
132
132
 
@@ -10,15 +10,186 @@ module Html2rss
10
10
  # supporting both single and multiple feed configurations.
11
11
  #
12
12
  # Configuration is validated during initialization.
13
- class Config
13
+ class Config # rubocop:disable Metrics/ClassLength
14
14
  # Raised when a configuration hash fails runtime validation.
15
15
  class InvalidConfig < Html2rss::Error; end
16
- extend ClassMethods
16
+ # Sentinel to differentiate omitted params from explicit `nil`.
17
+ UNSET = Object.new.freeze
18
+
19
+ class << self
20
+ ##
21
+ # Returns the exported JSON Schema for html2rss configuration.
22
+ #
23
+ # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
24
+ def json_schema
25
+ Schema.json_schema
26
+ end
27
+
28
+ ##
29
+ # Returns the exported JSON Schema as JSON.
30
+ #
31
+ # @param pretty [Boolean] whether to pretty-print the JSON output
32
+ # @return [String] serialized JSON Schema
33
+ def json_schema_json(pretty: true)
34
+ pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
35
+ end
36
+
37
+ ##
38
+ # Validates a configuration hash with the runtime validator.
39
+ #
40
+ # @param config [Hash{Symbol => Object}] the configuration hash
41
+ # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
42
+ # @return [Dry::Validation::Result] validation result after defaults are applied
43
+ def validate(config, params: UNSET)
44
+ prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
45
+
46
+ Validator.new.call(prepared_config)
47
+ rescue DynamicParams::ParamsMissing => error
48
+ prepared_config = prepare_for_validation(HashUtil.deep_symbolize_keys(config, context: 'config'))
49
+ prepared_config[:dynamic_params_error] = error.message
50
+
51
+ Validator.new.call(prepared_config)
52
+ end
53
+
54
+ ##
55
+ # Returns the packaged JSON Schema file path.
56
+ #
57
+ # @return [String] absolute path to the packaged JSON Schema file
58
+ def schema_path
59
+ Schema.path
60
+ end
61
+
62
+ ##
63
+ # Loads and validates a YAML configuration file.
64
+ #
65
+ # @param file [String] the YAML file to load
66
+ # @param feed_name [String, nil] optional feed name for multi-feed files
67
+ # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
68
+ # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
69
+ # @return [Dry::Validation::Result] validation result after defaults are applied
70
+ def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
71
+ validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
72
+ end
73
+
74
+ ##
75
+ # Loads the feed configuration from a YAML file.
76
+ #
77
+ # Supports multiple feeds defined under the specified key (default :feeds).
78
+ #
79
+ # @param file [String] the YAML file to load.
80
+ # @param feed_name [String, nil] the feed name when using multiple feeds.
81
+ # @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
82
+ # @return [Hash{Symbol => Object}] the configuration hash.
83
+ # @raise [ArgumentError] if the file doesn't exist or feed is not found.
84
+ # rubocop:disable Metrics/MethodLength
85
+ def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
86
+ raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
87
+ raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
88
+
89
+ yaml = YAML.safe_load_file(file, symbolize_names: true)
90
+
91
+ return yaml unless yaml.key?(multiple_feeds_key)
92
+
93
+ unless feed_name
94
+ available_feeds = yaml.fetch(multiple_feeds_key).keys.join(', ')
95
+ raise ArgumentError,
96
+ "Feed name is required under `#{multiple_feeds_key}`. Available feeds: #{available_feeds}"
97
+ end
98
+
99
+ config = yaml.dig(multiple_feeds_key, feed_name.to_sym)
100
+ raise ArgumentError, "Feed '#{feed_name}' not found under `#{multiple_feeds_key}` key." unless config
101
+
102
+ MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
103
+ end
104
+ # rubocop:enable Metrics/MethodLength
105
+
106
+ ##
107
+ # Processes the provided configuration hash, applying dynamic parameters if given,
108
+ # and returns a new configuration object.
109
+ #
110
+ # @param config [Hash{Symbol => Object}] the configuration hash.
111
+ # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting.
112
+ # @return [Html2rss::Config] the configuration object.
113
+ def from_hash(config, params: UNSET)
114
+ new(resolve_effective_config(config, params:))
115
+ end
116
+
117
+ ##
118
+ # Builds a top-level auto-source feed config for the public shortcut APIs.
119
+ #
120
+ # @param url [String] source page URL
121
+ # @param items_selector [String, nil] optional selector hint for item extraction
122
+ # @param request_controls [Html2rss::Config::RequestControls, nil] explicit request controls to write
123
+ # @return [Hash{Symbol => Object}] feed config hash ready for {from_hash}
124
+ def auto_source_config(url:, items_selector: nil, request_controls: nil)
125
+ config = {
126
+ channel: default_config[:channel].merge(url:),
127
+ auto_source: AutoSource::DEFAULT_CONFIG
128
+ }
129
+
130
+ request_controls ||= RequestControls.new
131
+ request_controls.apply_to(config)
132
+
133
+ config[:selectors] = { items: { selector: items_selector, enhance: true } } if items_selector
134
+ config
135
+ end
136
+
137
+ ##
138
+ # Provides a default configuration.
139
+ #
140
+ # @return [Hash{Symbol => Object}] a hash with default configuration values.
141
+ def default_config
142
+ {
143
+ strategy: default_strategy_name,
144
+ request: default_request_config,
145
+ channel: { time_zone: 'UTC' },
146
+ headers: RequestHeaders.browser_defaults,
147
+ stylesheets: Html2rss.defaults.stylesheets || []
148
+ }
149
+ end
150
+
151
+ # @return [Symbol] the default feed-level strategy plan (+:auto+ or concrete)
152
+ def default_strategy_name
153
+ Html2rss.defaults.default_strategy || :auto
154
+ end
155
+
156
+ private
157
+
158
+ def default_request_config
159
+ {
160
+ max_redirects: RequestService::Policy::DEFAULTS[:max_redirects],
161
+ max_requests: RequestService::Policy::DEFAULTS[:max_requests],
162
+ total_timeout_seconds: RequestService::Policy::DEFAULTS[:total_timeout_seconds]
163
+ }
164
+ end
165
+
166
+ def resolve_effective_config(config, params:)
167
+ cfg = HashUtil.deep_symbolize_keys(config, context: 'config')
168
+ p = parameter_defaults(cfg)
169
+ p.merge!(HashUtil.deep_symbolize_keys(params, context: 'params')) unless params.equal?(UNSET) || params.nil?
170
+
171
+ cfg[:headers] = DynamicParams.call(cfg[:headers], p) if cfg[:headers]
172
+ cfg[:channel] = DynamicParams.call(cfg[:channel], p) if cfg[:channel]
173
+ cfg
174
+ end
175
+
176
+ def parameter_defaults(config)
177
+ config.fetch(:parameters, {})
178
+ .filter_map do |name, definition|
179
+ [name, definition[:default]] if definition.is_a?(Hash) && definition.key?(:default)
180
+ end
181
+ .to_h
182
+ end
183
+
184
+ def prepare_for_validation(config)
185
+ Config::Preparer.new.call(HashUtil.deep_dup(config))
186
+ end
187
+ end
17
188
 
18
189
  ##
19
190
  # Initializes the configuration object.
20
191
  #
21
- # Processes deprecated attributes, applies default values, and validates the configuration.
192
+ # Applies default values and validates the configuration.
22
193
  #
23
194
  # @param config [Hash{Symbol => Object}] the configuration hash.
24
195
  # @raise [InvalidConfig] if the configuration fails validation.
@@ -54,7 +225,7 @@ module Html2rss
54
225
  end
55
226
 
56
227
  ##
57
- # @return [Html2rss::RequestControls] request controls with provenance
228
+ # @return [Html2rss::Config::RequestControls] request controls with provenance
58
229
  attr_reader :request_controls
59
230
 
60
231
  # @return [Hash{String => String}] normalized HTTP headers
@@ -82,11 +253,10 @@ module Html2rss
82
253
  class Preparer
83
254
  ##
84
255
  # @param config [Hash{Symbol => Object}] raw config input
85
- # @return [Hash{Symbol => Object}] config with defaults and deprecations applied
256
+ # @return [Hash{Symbol => Object}] config with defaults applied
86
257
  def call(config)
87
258
  config = config.dup if config.frozen?
88
259
 
89
- config = handle_deprecated_channel_attributes(config)
90
260
  config = apply_default_config(config)
91
261
  config = apply_default_selectors_config(config) if config[:selectors]
92
262
  config = apply_default_auto_source_config(config) if config[:auto_source]
@@ -96,19 +266,6 @@ module Html2rss
96
266
 
97
267
  private
98
268
 
99
- def handle_deprecated_channel_attributes(config)
100
- { strategy: Config.default_strategy_name, headers: {} }.each_pair do |key, default_value|
101
- if !config[key] && (value = config.dig(:channel, key))
102
- Log.warn("The `channel.#{key}` key is deprecated. Please move the definition of `#{key}` to the top level.")
103
- config[key] = value
104
- end
105
-
106
- config[key] ||= default_value
107
- end
108
-
109
- config
110
- end
111
-
112
269
  def apply_default_config(config)
113
270
  HashUtil.deep_merge(Config.default_config, config)
114
271
  end
@@ -2,8 +2,8 @@
2
2
 
3
3
  module Html2rss
4
4
  ##
5
- # Global configuration defaults for the Html2rss gem.
6
- class Configuration
5
+ # Global defaults for the Html2rss gem.
6
+ class Defaults
7
7
  # The valid symbol log levels.
8
8
  VALID_LOG_LEVELS = %i[debug info warn error fatal unknown].freeze
9
9
 
@@ -29,7 +29,7 @@ module Html2rss
29
29
  attr_reader :stylesheets
30
30
 
31
31
  ##
32
- # Initializes a new Configuration instance with defaults.
32
+ # Initializes a new Defaults instance.
33
33
  def initialize
34
34
  @logger_formatter = proc do |severity, datetime, _progname, msg|
35
35
  "#{datetime} [#{severity}] #{msg}\n"
@@ -93,11 +93,13 @@ module Html2rss
93
93
  end
94
94
 
95
95
  ##
96
- # Sets the default strategy.
96
+ # Sets the default feed-level strategy plan (+:auto+ or a concrete transport strategy).
97
97
  #
98
- # @param strategy [Symbol, String, nil] the strategy name
99
- # @return [Symbol, nil] the normalized strategy name
100
- # @raise [ArgumentError] if the strategy is not registered
98
+ # +:auto+ is not a {RequestService} adapter it is resolved by {FeedPipeline::StrategyPlan}.
99
+ #
100
+ # @param strategy [Symbol, String, nil] the strategy plan name
101
+ # @return [Symbol, nil] the normalized strategy plan name
102
+ # @raise [ArgumentError] if the strategy plan is unknown
101
103
  def default_strategy=(strategy)
102
104
  if strategy.nil?
103
105
  @default_strategy = nil
@@ -107,7 +109,7 @@ module Html2rss
107
109
  end
108
110
 
109
111
  normalized = strategy.to_sym
110
- raise ArgumentError, "unknown strategy: #{strategy}" unless RequestService.strategy_registered?(normalized)
112
+ raise ArgumentError, "unknown strategy: #{strategy}" unless FeedPipeline::StrategyPlan.valid?(normalized)
111
113
 
112
114
  @default_strategy = normalized
113
115
  end
@@ -148,9 +150,9 @@ module Html2rss
148
150
  protected
149
151
 
150
152
  ##
151
- # Copy constructor for duplicating configuration.
153
+ # Copy constructor for duplicating defaults.
152
154
  #
153
- # @param other [Html2rss::Configuration] the original configuration
155
+ # @param other [Html2rss::Defaults] the original defaults
154
156
  # @return [void]
155
157
  def initialize_copy(other)
156
158
  super
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ module FeedBuilder
5
+ ##
6
+ # Presentation rules for rendering an {Html2rss::Article} into feed wire formats.
7
+ #
8
+ # Owns description enrichment ({Html::Rendering::DescriptionBuilder}) and RSS
9
+ # non-image enclosure selection. {Article} keeps raw extracted fields only.
10
+ module ItemPresentation
11
+ class << self
12
+ ##
13
+ # @param article [Html2rss::Article]
14
+ # @return [String, nil] sanitized description HTML/text for feed item bodies
15
+ def description_for(article)
16
+ Html::Rendering::DescriptionBuilder.new(
17
+ base: article.description,
18
+ title: article.title,
19
+ url: article.url,
20
+ enclosures: article.enclosures,
21
+ image: article.image
22
+ ).call
23
+ end
24
+
25
+ ##
26
+ # First non-image enclosure for RSS +enclosure+ (images stay on description / JSON Feed).
27
+ #
28
+ # @param article [Html2rss::Article]
29
+ # @return [Html2rss::Article::Enclosure, nil]
30
+ def rss_enclosure_for(article)
31
+ article.enclosures.find { |enc| !image_mime_type?(enc.type) }
32
+ end
33
+
34
+ private
35
+
36
+ def image_mime_type?(type)
37
+ type.to_s.downcase.start_with?('image/')
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end