html2rss 0.22.2 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +11 -108
- data/lib/html2rss/{articles → article}/deduplicator.rb +3 -4
- data/lib/html2rss/{rss_builder → article}/enclosure.rb +6 -22
- data/lib/html2rss/article.rb +206 -0
- data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb +80 -0
- data/lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb +86 -0
- data/lib/html2rss/auto_source/discovery/dom_clustering.rb +119 -0
- data/lib/html2rss/auto_source/discovery/list_candidates.rb +94 -0
- data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb +219 -0
- data/lib/html2rss/auto_source/discovery/semantic_containers.rb +71 -0
- data/lib/html2rss/auto_source/discovery/sitemap.rb +159 -0
- data/lib/html2rss/auto_source/discovery.rb +14 -0
- data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb +28 -0
- data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb +106 -0
- data/lib/html2rss/auto_source/link_heuristics/container_signals.rb +80 -0
- data/lib/html2rss/auto_source/link_heuristics/destination_facts.rb +42 -0
- data/lib/html2rss/auto_source/link_heuristics/href_extractor.rb +38 -0
- data/lib/html2rss/auto_source/link_heuristics/path_classifier.rb +221 -0
- data/lib/html2rss/auto_source/link_heuristics/text_classifier.rb +66 -0
- data/lib/html2rss/auto_source/link_heuristics.rb +139 -0
- data/lib/html2rss/auto_source/scraper/html.rb +22 -33
- data/lib/html2rss/auto_source/scraper/json_state.rb +1 -1
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +163 -0
- data/lib/html2rss/auto_source/scraper/microformats2.rb +146 -0
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +21 -2
- data/lib/html2rss/auto_source/scraper/semantic_html/{deduplicator.rb → entry_deduplicator.rb} +3 -3
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +23 -171
- data/lib/html2rss/auto_source/scraper/sitemap.rb +153 -0
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb +110 -0
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -78
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +9 -0
- data/lib/html2rss/auto_source/scraper.rb +10 -3
- data/lib/html2rss/auto_source.rb +30 -35
- data/lib/html2rss/channel.rb +118 -0
- data/lib/html2rss/cli.rb +5 -6
- data/lib/html2rss/config/auto_source_contract.rb +51 -0
- data/lib/html2rss/config/request_controls.rb +136 -0
- data/lib/html2rss/config/request_headers.rb +1 -1
- data/lib/html2rss/config/schema.rb +65 -114
- data/lib/html2rss/{selectors/config.rb → config/selectors_validator.rb} +56 -7
- data/lib/html2rss/config/validator.rb +6 -6
- data/lib/html2rss/config.rb +176 -19
- data/lib/html2rss/{configuration.rb → defaults.rb} +12 -10
- data/lib/html2rss/feed_builder/json_feed/item.rb +96 -0
- data/lib/html2rss/feed_builder/json_feed.rb +61 -0
- data/lib/html2rss/feed_builder/rss/stylesheet.rb +65 -0
- data/lib/html2rss/feed_builder/rss.rb +116 -0
- data/lib/html2rss/feed_builder.rb +23 -0
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +59 -42
- data/lib/html2rss/feed_pipeline/runtime_policy.rb +108 -0
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +64 -0
- data/lib/html2rss/feed_pipeline.rb +52 -65
- data/lib/html2rss/html/article_extractor/category_extractor.rb +123 -0
- data/lib/html2rss/html/article_extractor/date_extractor.rb +22 -0
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +95 -0
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +54 -0
- data/lib/html2rss/html/article_extractor/id_generator.rb +69 -0
- data/lib/html2rss/html/article_extractor/image_extractor.rb +60 -0
- data/lib/html2rss/html/article_extractor.rb +134 -0
- data/lib/html2rss/html/navigator/text_extractor.rb +79 -0
- data/lib/html2rss/html/navigator.rb +146 -0
- data/lib/html2rss/html/rendering/audio_renderer.rb +38 -0
- data/lib/html2rss/html/rendering/description_builder.rb +89 -0
- data/lib/html2rss/html/rendering/image_renderer.rb +43 -0
- data/lib/html2rss/html/rendering/media_renderer.rb +39 -0
- data/lib/html2rss/html/rendering/pdf_renderer.rb +36 -0
- data/lib/html2rss/html/rendering/video_renderer.rb +38 -0
- data/lib/html2rss/html/rendering.rb +25 -0
- data/lib/html2rss/html.rb +8 -0
- data/lib/html2rss/request_service/blocked_surface.rb +67 -0
- data/lib/html2rss/request_service/botasaurus_contract.rb +0 -4
- data/lib/html2rss/request_service/botasaurus_strategy.rb +9 -27
- data/lib/html2rss/request_service/browserless_strategy.rb +2 -23
- data/lib/html2rss/request_service/budget.rb +90 -16
- data/lib/html2rss/request_service/context.rb +1 -3
- data/lib/html2rss/request_service/faraday_strategy.rb +12 -26
- data/lib/html2rss/request_service/local_file_strategy.rb +12 -4
- data/lib/html2rss/request_service/network_guard.rb +125 -0
- data/lib/html2rss/request_service/policy.rb +10 -82
- data/lib/html2rss/request_service/puppet_commander/navigation_guards.rb +148 -0
- data/lib/html2rss/request_service/puppet_commander/preload_runner.rb +86 -0
- data/lib/html2rss/request_service/puppet_commander.rb +22 -191
- data/lib/html2rss/request_service/response_guard.rb +1 -1
- data/lib/html2rss/request_service/strategy.rb +102 -6
- data/lib/html2rss/request_service.rb +7 -2
- data/lib/html2rss/request_session/pager/base.rb +83 -0
- data/lib/html2rss/request_session/pager/custom_selector.rb +39 -0
- data/lib/html2rss/request_session/pager/json_cursor.rb +78 -0
- data/lib/html2rss/request_session/pager/offset.rb +48 -0
- data/lib/html2rss/request_session/pager/rel_next.rb +23 -0
- data/lib/html2rss/request_session/pager/url_template.rb +48 -0
- data/lib/html2rss/request_session/pager.rb +75 -0
- data/lib/html2rss/request_session.rb +25 -16
- data/lib/html2rss/selectors/item_scope.rb +30 -0
- data/lib/html2rss/selectors/post_processors/base.rb +1 -1
- data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +0 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +0 -2
- data/lib/html2rss/selectors/post_processors/template.rb +7 -3
- data/lib/html2rss/selectors.rb +63 -65
- data/lib/html2rss/version.rb +1 -3
- data/lib/html2rss.rb +19 -19
- data/schema/html2rss-config.schema.json +109 -9
- metadata +68 -42
- data/lib/html2rss/auto_source/scraper/html/class_clustering.rb +0 -196
- data/lib/html2rss/auto_source/scraper/link_heuristics.rb +0 -401
- data/lib/html2rss/auto_source/scraper/semantic_html/anchor_selector.rb +0 -49
- data/lib/html2rss/blocked_surface.rb +0 -65
- data/lib/html2rss/category_extractor.rb +0 -116
- data/lib/html2rss/config/class_methods.rb +0 -183
- data/lib/html2rss/html_extractor/date_extractor.rb +0 -20
- data/lib/html2rss/html_extractor/enclosure_extractor.rb +0 -91
- data/lib/html2rss/html_extractor/heading_extractor.rb +0 -50
- data/lib/html2rss/html_extractor/id_generator.rb +0 -67
- data/lib/html2rss/html_extractor/image_extractor.rb +0 -58
- data/lib/html2rss/html_extractor/list_candidates.rb +0 -88
- data/lib/html2rss/html_extractor/semantic_anchor_candidates.rb +0 -260
- data/lib/html2rss/html_extractor/semantic_containers.rb +0 -69
- data/lib/html2rss/html_extractor/text_extractor.rb +0 -77
- data/lib/html2rss/html_extractor.rb +0 -190
- data/lib/html2rss/html_navigator.rb +0 -71
- data/lib/html2rss/json_feed_builder/item.rb +0 -94
- data/lib/html2rss/json_feed_builder.rb +0 -59
- data/lib/html2rss/rendering/audio_renderer.rb +0 -36
- data/lib/html2rss/rendering/description_builder.rb +0 -87
- data/lib/html2rss/rendering/image_renderer.rb +0 -41
- data/lib/html2rss/rendering/media_renderer.rb +0 -37
- data/lib/html2rss/rendering/pdf_renderer.rb +0 -34
- data/lib/html2rss/rendering/video_renderer.rb +0 -36
- data/lib/html2rss/rendering.rb +0 -23
- data/lib/html2rss/request_controls.rb +0 -133
- data/lib/html2rss/request_session/rel_next_pager.rb +0 -70
- data/lib/html2rss/request_session/runtime_input.rb +0 -71
- data/lib/html2rss/request_session/runtime_policy.rb +0 -84
- data/lib/html2rss/rss_builder/article.rb +0 -208
- data/lib/html2rss/rss_builder/channel.rb +0 -118
- data/lib/html2rss/rss_builder/stylesheet.rb +0 -63
- data/lib/html2rss/rss_builder.rb +0 -101
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
class Config
|
|
5
5
|
##
|
|
6
|
-
#
|
|
6
|
+
# JSON export adapter from the runtime validators to the packaged schema artifact.
|
|
7
7
|
module Schema
|
|
8
8
|
module_function
|
|
9
9
|
|
|
@@ -60,7 +60,8 @@ module Html2rss
|
|
|
60
60
|
def call
|
|
61
61
|
schema = validator_schema
|
|
62
62
|
apply_top_level(schema)
|
|
63
|
-
|
|
63
|
+
schema.fetch(:properties).merge!(overlay)
|
|
64
|
+
schema.fetch(:properties).delete(:dynamic_params_error)
|
|
64
65
|
DeepStringifier.call(schema)
|
|
65
66
|
end
|
|
66
67
|
|
|
@@ -78,124 +79,74 @@ module Html2rss
|
|
|
78
79
|
]
|
|
79
80
|
end
|
|
80
81
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
# @return [Hash{Symbol => Object}] schema fragment for headers
|
|
107
|
-
def headers
|
|
108
|
-
{
|
|
109
|
-
type: 'object',
|
|
110
|
-
description: 'HTTP headers applied to every request.',
|
|
111
|
-
additionalProperties: { type: 'string' }
|
|
112
|
-
}
|
|
113
|
-
end
|
|
114
|
-
|
|
115
|
-
# @return [Hash{Symbol => Object}] schema fragment for stylesheet definitions
|
|
116
|
-
def stylesheets
|
|
117
|
-
{
|
|
118
|
-
type: 'array',
|
|
119
|
-
description: 'Collection of stylesheets to attach to the RSS feed.',
|
|
120
|
-
items: Html2rss::Config::Validator::StylesheetConfig.json_schema(loose: true)
|
|
121
|
-
}
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
# @return [Hash{Symbol => Object}] schema fragment for auto_source configuration
|
|
125
|
-
def auto_source
|
|
126
|
-
schema = Html2rss::AutoSource::Config.json_schema(loose: true)
|
|
127
|
-
schema[:default] = DeepStringifier.call(Html2rss::AutoSource::DEFAULT_CONFIG)
|
|
128
|
-
schema
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
# @return [Hash{Symbol => Object}] schema fragment for selectors configuration
|
|
132
|
-
def selectors
|
|
133
|
-
Selectors.schema
|
|
134
|
-
end
|
|
135
|
-
end
|
|
136
|
-
|
|
137
|
-
##
|
|
138
|
-
# Provides schema fragments that document selector configuration.
|
|
139
|
-
module Selectors
|
|
140
|
-
module_function
|
|
141
|
-
|
|
142
|
-
# Pattern used for dynamic selector keys excluding reserved selector names.
|
|
143
|
-
RESERVED_SELECTOR_PATTERN = '^(?!items$|enclosure$|guid$|categories$).+$'
|
|
144
|
-
|
|
145
|
-
# @return [Hash{Symbol => Object}] schema fragment for selectors root object
|
|
146
|
-
def schema
|
|
147
|
-
{
|
|
148
|
-
type: 'object',
|
|
149
|
-
description: 'Selectors used to extract article attributes.',
|
|
150
|
-
properties: selector_properties,
|
|
151
|
-
patternProperties: pattern_properties,
|
|
152
|
-
additionalProperties: true
|
|
82
|
+
# rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
|
|
83
|
+
def overlay
|
|
84
|
+
items_schema = Html2rss::Config::SelectorsValidator::Items.new.schema.json_schema(loose: true)
|
|
85
|
+
items_schema[:properties][:pagination] = {
|
|
86
|
+
description: 'Pagination configuration or maximum page count integer.',
|
|
87
|
+
oneOf: [
|
|
88
|
+
{ type: 'integer', exclusiveMinimum: 0 },
|
|
89
|
+
{
|
|
90
|
+
type: 'object',
|
|
91
|
+
properties: {
|
|
92
|
+
strategy: { type: 'string', enum: Html2rss::RequestSession::Pager.strategy_names },
|
|
93
|
+
max_pages: { type: 'integer', exclusiveMinimum: 0 },
|
|
94
|
+
selector: { type: 'string' },
|
|
95
|
+
param: { type: 'string' },
|
|
96
|
+
start_page: { type: 'integer' },
|
|
97
|
+
step: { type: 'integer', exclusiveMinimum: 0 },
|
|
98
|
+
start_offset: { type: 'integer' },
|
|
99
|
+
increment: { type: 'integer', exclusiveMinimum: 0 },
|
|
100
|
+
cursor_path: { type: 'string' },
|
|
101
|
+
next_url_path: { type: 'string' }
|
|
102
|
+
},
|
|
103
|
+
additionalProperties: true
|
|
104
|
+
}
|
|
105
|
+
]
|
|
153
106
|
}
|
|
154
|
-
end
|
|
155
107
|
|
|
156
|
-
# rubocop:disable Layout/LineLength
|
|
157
|
-
# @return [Hash{Symbol => Object}] schema map for reserved selector properties
|
|
158
|
-
def selector_properties
|
|
159
108
|
{
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
109
|
+
strategy: {
|
|
110
|
+
type: 'string',
|
|
111
|
+
not: { type: 'null' }
|
|
112
|
+
},
|
|
113
|
+
headers: {
|
|
114
|
+
type: 'object',
|
|
115
|
+
description: 'HTTP headers applied to every request.',
|
|
116
|
+
additionalProperties: { type: 'string' }
|
|
117
|
+
},
|
|
118
|
+
stylesheets: {
|
|
119
|
+
type: 'array',
|
|
120
|
+
description: 'Collection of stylesheets to attach to the RSS feed.',
|
|
121
|
+
items: Html2rss::Config::Validator::StylesheetConfig.json_schema(loose: true)
|
|
122
|
+
},
|
|
123
|
+
auto_source: Html2rss::Config::AutoSourceContract.json_schema(loose: true).merge(
|
|
124
|
+
default: DeepStringifier.call(Html2rss::AutoSource::DEFAULT_CONFIG)
|
|
125
|
+
),
|
|
126
|
+
selectors: {
|
|
127
|
+
type: 'object',
|
|
128
|
+
description: 'Selectors used to extract article attributes.',
|
|
129
|
+
properties: {
|
|
130
|
+
items: items_schema.merge(
|
|
131
|
+
description: 'Defines the items selector and optional enhancement settings.'
|
|
132
|
+
),
|
|
133
|
+
enclosure: Html2rss::Config::SelectorsValidator::Enclosure.new.schema.json_schema(loose: true).merge(
|
|
134
|
+
description: 'Describes enclosure extraction settings.'
|
|
135
|
+
),
|
|
136
|
+
guid: reference_array('List of selector keys used to build the GUID. Each entry must reference a sibling selector key; runtime validation enforces those references.'),
|
|
137
|
+
categories: reference_array('List of selector keys whose values will be used as categories. Each entry must reference a sibling selector key; runtime validation enforces those references.')
|
|
138
|
+
},
|
|
139
|
+
patternProperties: {
|
|
140
|
+
'^(?!items$|enclosure$|guid$|categories$).+$' => Html2rss::Config::SelectorsValidator::Selector.new.schema.json_schema(loose: true).merge(
|
|
141
|
+
description: 'Dynamic selector definition keyed by attribute name.'
|
|
142
|
+
)
|
|
143
|
+
},
|
|
144
|
+
additionalProperties: true
|
|
145
|
+
}
|
|
164
146
|
}
|
|
165
147
|
end
|
|
166
|
-
# rubocop:enable Layout/LineLength
|
|
167
|
-
|
|
168
|
-
# @return [Hash{String => Object}] schema map for dynamic selector keys
|
|
169
|
-
def pattern_properties
|
|
170
|
-
{ RESERVED_SELECTOR_PATTERN => dynamic_selector_schema }
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
# @return [Hash{Symbol => Object}] schema fragment for dynamic selector entries
|
|
174
|
-
def dynamic_selector_schema
|
|
175
|
-
Html2rss::Selectors::Config::Selector.new.schema.json_schema(loose: true).merge(
|
|
176
|
-
description: 'Dynamic selector definition keyed by attribute name.'
|
|
177
|
-
)
|
|
178
|
-
end
|
|
179
|
-
|
|
180
|
-
# @return [Hash{Symbol => Object}] schema fragment for `items` selector configuration
|
|
181
|
-
def items_schema
|
|
182
|
-
Html2rss::Selectors::Config::Items.new.schema.json_schema(loose: true).merge(
|
|
183
|
-
description: 'Defines the items selector and optional enhancement settings.'
|
|
184
|
-
)
|
|
185
|
-
end
|
|
186
|
-
|
|
187
|
-
# @return [Hash{Symbol => Object}] schema fragment for `enclosure` selector configuration
|
|
188
|
-
def enclosure_schema
|
|
189
|
-
Html2rss::Selectors::Config::Enclosure.new.schema.json_schema(loose: true).merge(
|
|
190
|
-
description: 'Describes enclosure extraction settings.'
|
|
191
|
-
)
|
|
192
|
-
end
|
|
148
|
+
# rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
|
|
193
149
|
|
|
194
|
-
# JSON Schema can enforce non-empty reference arrays, while runtime
|
|
195
|
-
# validation remains authoritative for checking that each entry points
|
|
196
|
-
# to an existing sibling selector key.
|
|
197
|
-
# @param description [String] human-readable description for the reference field
|
|
198
|
-
# @return [Hash{Symbol => Object}] JSON schema fragment for selector references
|
|
199
150
|
def reference_array(description)
|
|
200
151
|
{
|
|
201
152
|
type: 'array',
|
|
@@ -3,23 +3,72 @@
|
|
|
3
3
|
require 'dry-validation'
|
|
4
4
|
|
|
5
5
|
module Html2rss
|
|
6
|
-
class
|
|
6
|
+
class Config
|
|
7
7
|
##
|
|
8
8
|
# Validates the configuration hash for :selectors.
|
|
9
|
-
class
|
|
9
|
+
class SelectorsValidator < Dry::Validation::Contract
|
|
10
10
|
# Required wrapper key used to validate dynamic selector names.
|
|
11
11
|
NESTING_KEY = :dynamic_keys_workaround
|
|
12
12
|
|
|
13
13
|
##
|
|
14
14
|
# Validates the configuration of the :items selector
|
|
15
15
|
class Items < Dry::Validation::Contract
|
|
16
|
+
# Supported pagination strategy names (sole source: RequestSession::Pager).
|
|
17
|
+
# @return [Array<String>]
|
|
18
|
+
STRATEGY_NAMES = Html2rss::RequestSession::Pager.strategy_names
|
|
19
|
+
|
|
16
20
|
params do
|
|
17
21
|
required(:selector).filled(:string)
|
|
18
22
|
optional(:order).filled(included_in?: %w[reverse])
|
|
19
23
|
optional(:enhance).filled(:bool?)
|
|
20
|
-
optional(:pagination)
|
|
21
|
-
|
|
24
|
+
optional(:pagination)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
rule(:pagination) do
|
|
28
|
+
next if value.nil?
|
|
29
|
+
|
|
30
|
+
if value.is_a?(Integer)
|
|
31
|
+
key.failure('integer must be greater than 0') if value <= 0
|
|
32
|
+
elsif value.is_a?(Hash)
|
|
33
|
+
val_hash = value.transform_keys(&:to_sym)
|
|
34
|
+
validate_pagination_hash(val_hash, key)
|
|
35
|
+
else
|
|
36
|
+
key.failure('must be an integer or a hash')
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def validate_pagination_hash(val_hash, key)
|
|
43
|
+
# Fail loud on explicit nil/non-positive: truthy checks would silently accept max_pages: nil.
|
|
44
|
+
if val_hash.key?(:max_pages) && (!val_hash[:max_pages].is_a?(Integer) || val_hash[:max_pages] <= 0)
|
|
45
|
+
key.failure('`max_pages` must be an integer greater than 0')
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
strategy = (val_hash[:strategy] || 'rel_next').to_s
|
|
49
|
+
unless STRATEGY_NAMES.include?(strategy)
|
|
50
|
+
key.failure("`strategy` must be one of: #{STRATEGY_NAMES.join(', ')}")
|
|
22
51
|
end
|
|
52
|
+
|
|
53
|
+
validate_strategy_fields(strategy, val_hash, key)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def validate_strategy_fields(strategy, val_hash, key)
|
|
57
|
+
validate_custom_selector(val_hash, key) if strategy == 'custom_selector'
|
|
58
|
+
validate_json_cursor(val_hash, key) if strategy == 'json_cursor'
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def validate_custom_selector(val_hash, key)
|
|
62
|
+
return unless val_hash[:selector].nil? || val_hash[:selector].to_s.strip.empty?
|
|
63
|
+
|
|
64
|
+
key.failure('`custom_selector` strategy requires `selector` to be specified')
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def validate_json_cursor(val_hash, key)
|
|
68
|
+
return unless (val_hash[:cursor_path].nil? || val_hash[:cursor_path].to_s.strip.empty?) &&
|
|
69
|
+
(val_hash[:next_url_path].nil? || val_hash[:next_url_path].to_s.strip.empty?)
|
|
70
|
+
|
|
71
|
+
key.failure('`json_cursor` strategy requires either `cursor_path` or `next_url_path`')
|
|
23
72
|
end
|
|
24
73
|
end
|
|
25
74
|
|
|
@@ -84,9 +133,9 @@ module Html2rss
|
|
|
84
133
|
value.each_pair do |selector_key, selector|
|
|
85
134
|
case selector_key.to_sym
|
|
86
135
|
when Selectors::ITEMS_SELECTOR_KEY
|
|
87
|
-
Items.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
|
|
136
|
+
Items.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
|
|
88
137
|
when :enclosure
|
|
89
|
-
Enclosure.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
|
|
138
|
+
Enclosure.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
|
|
90
139
|
when :guid, :categories
|
|
91
140
|
unless selector.is_a?(Array)
|
|
92
141
|
key(selector_key).failure("`#{selector_key}` must be an array")
|
|
@@ -102,7 +151,7 @@ module Html2rss
|
|
|
102
151
|
end
|
|
103
152
|
else
|
|
104
153
|
# From here on, the selector is found under its "dynamic" selector_key
|
|
105
|
-
Selector.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
|
|
154
|
+
Selector.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
|
|
106
155
|
end
|
|
107
156
|
end
|
|
108
157
|
end
|
|
@@ -10,11 +10,11 @@ module Html2rss
|
|
|
10
10
|
# URI format used for channel URL validation.
|
|
11
11
|
URI_REGEXP = Url::URI_REGEXP
|
|
12
12
|
# Allowed stylesheet MIME types.
|
|
13
|
-
STYLESHEET_TYPES =
|
|
13
|
+
STYLESHEET_TYPES = FeedBuilder::Rss::Stylesheet::TYPES
|
|
14
14
|
# Optional language/region format (`en` or `en-US`).
|
|
15
15
|
LANGUAGE_FORMAT_REGEX = /\A[a-z]{2}(-[A-Z]{2})?\z/
|
|
16
|
-
# Baseline strategy enum
|
|
17
|
-
BASE_STRATEGY_OPTIONS =
|
|
16
|
+
# Baseline strategy-plan enum (:auto plus concrete RequestService strategies).
|
|
17
|
+
BASE_STRATEGY_OPTIONS = Html2rss::FeedPipeline::StrategyPlan.accepted_names.freeze
|
|
18
18
|
|
|
19
19
|
# Contract for the top-level `channel` section.
|
|
20
20
|
ChannelConfig = Dry::Schema.Params do
|
|
@@ -91,7 +91,7 @@ module Html2rss
|
|
|
91
91
|
required(:channel).hash(ChannelConfig)
|
|
92
92
|
optional(:headers).hash
|
|
93
93
|
optional(:stylesheets).array(StylesheetConfig)
|
|
94
|
-
optional(:auto_source).hash(
|
|
94
|
+
optional(:auto_source).hash(Config::AutoSourceContract)
|
|
95
95
|
optional(:selectors).hash
|
|
96
96
|
optional(:dynamic_params_error).maybe(:string)
|
|
97
97
|
optional(:request).hash(RequestConfig)
|
|
@@ -111,7 +111,7 @@ module Html2rss
|
|
|
111
111
|
|
|
112
112
|
rule(:strategy) do
|
|
113
113
|
next if value.nil?
|
|
114
|
-
next if
|
|
114
|
+
next if Html2rss::FeedPipeline::StrategyPlan.valid?(value)
|
|
115
115
|
|
|
116
116
|
key.failure("must be one of: #{BASE_STRATEGY_OPTIONS.join(', ')}")
|
|
117
117
|
end
|
|
@@ -126,7 +126,7 @@ module Html2rss
|
|
|
126
126
|
rule(:selectors) do
|
|
127
127
|
next unless value
|
|
128
128
|
|
|
129
|
-
errors =
|
|
129
|
+
errors = Config::SelectorsValidator.call(value).errors
|
|
130
130
|
errors.each { |error| key(:selectors).failure(error.text) } unless errors.empty?
|
|
131
131
|
end
|
|
132
132
|
|
data/lib/html2rss/config.rb
CHANGED
|
@@ -10,15 +10,186 @@ module Html2rss
|
|
|
10
10
|
# supporting both single and multiple feed configurations.
|
|
11
11
|
#
|
|
12
12
|
# Configuration is validated during initialization.
|
|
13
|
-
class Config
|
|
13
|
+
class Config # rubocop:disable Metrics/ClassLength
|
|
14
14
|
# Raised when a configuration hash fails runtime validation.
|
|
15
15
|
class InvalidConfig < Html2rss::Error; end
|
|
16
|
-
|
|
16
|
+
# Sentinel to differentiate omitted params from explicit `nil`.
|
|
17
|
+
UNSET = Object.new.freeze
|
|
18
|
+
|
|
19
|
+
class << self
|
|
20
|
+
##
|
|
21
|
+
# Returns the exported JSON Schema for html2rss configuration.
|
|
22
|
+
#
|
|
23
|
+
# @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
|
|
24
|
+
def json_schema
|
|
25
|
+
Schema.json_schema
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
##
|
|
29
|
+
# Returns the exported JSON Schema as JSON.
|
|
30
|
+
#
|
|
31
|
+
# @param pretty [Boolean] whether to pretty-print the JSON output
|
|
32
|
+
# @return [String] serialized JSON Schema
|
|
33
|
+
def json_schema_json(pretty: true)
|
|
34
|
+
pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
##
|
|
38
|
+
# Validates a configuration hash with the runtime validator.
|
|
39
|
+
#
|
|
40
|
+
# @param config [Hash{Symbol => Object}] the configuration hash
|
|
41
|
+
# @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
|
|
42
|
+
# @return [Dry::Validation::Result] validation result after defaults are applied
|
|
43
|
+
def validate(config, params: UNSET)
|
|
44
|
+
prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
|
|
45
|
+
|
|
46
|
+
Validator.new.call(prepared_config)
|
|
47
|
+
rescue DynamicParams::ParamsMissing => error
|
|
48
|
+
prepared_config = prepare_for_validation(HashUtil.deep_symbolize_keys(config, context: 'config'))
|
|
49
|
+
prepared_config[:dynamic_params_error] = error.message
|
|
50
|
+
|
|
51
|
+
Validator.new.call(prepared_config)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
##
|
|
55
|
+
# Returns the packaged JSON Schema file path.
|
|
56
|
+
#
|
|
57
|
+
# @return [String] absolute path to the packaged JSON Schema file
|
|
58
|
+
def schema_path
|
|
59
|
+
Schema.path
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
##
|
|
63
|
+
# Loads and validates a YAML configuration file.
|
|
64
|
+
#
|
|
65
|
+
# @param file [String] the YAML file to load
|
|
66
|
+
# @param feed_name [String, nil] optional feed name for multi-feed files
|
|
67
|
+
# @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
|
|
68
|
+
# @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
|
|
69
|
+
# @return [Dry::Validation::Result] validation result after defaults are applied
|
|
70
|
+
def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
|
|
71
|
+
validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
##
|
|
75
|
+
# Loads the feed configuration from a YAML file.
|
|
76
|
+
#
|
|
77
|
+
# Supports multiple feeds defined under the specified key (default :feeds).
|
|
78
|
+
#
|
|
79
|
+
# @param file [String] the YAML file to load.
|
|
80
|
+
# @param feed_name [String, nil] the feed name when using multiple feeds.
|
|
81
|
+
# @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
|
|
82
|
+
# @return [Hash{Symbol => Object}] the configuration hash.
|
|
83
|
+
# @raise [ArgumentError] if the file doesn't exist or feed is not found.
|
|
84
|
+
# rubocop:disable Metrics/MethodLength
|
|
85
|
+
def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
|
|
86
|
+
raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
|
|
87
|
+
raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
|
|
88
|
+
|
|
89
|
+
yaml = YAML.safe_load_file(file, symbolize_names: true)
|
|
90
|
+
|
|
91
|
+
return yaml unless yaml.key?(multiple_feeds_key)
|
|
92
|
+
|
|
93
|
+
unless feed_name
|
|
94
|
+
available_feeds = yaml.fetch(multiple_feeds_key).keys.join(', ')
|
|
95
|
+
raise ArgumentError,
|
|
96
|
+
"Feed name is required under `#{multiple_feeds_key}`. Available feeds: #{available_feeds}"
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
config = yaml.dig(multiple_feeds_key, feed_name.to_sym)
|
|
100
|
+
raise ArgumentError, "Feed '#{feed_name}' not found under `#{multiple_feeds_key}` key." unless config
|
|
101
|
+
|
|
102
|
+
MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
|
|
103
|
+
end
|
|
104
|
+
# rubocop:enable Metrics/MethodLength
|
|
105
|
+
|
|
106
|
+
##
|
|
107
|
+
# Processes the provided configuration hash, applying dynamic parameters if given,
|
|
108
|
+
# and returns a new configuration object.
|
|
109
|
+
#
|
|
110
|
+
# @param config [Hash{Symbol => Object}] the configuration hash.
|
|
111
|
+
# @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting.
|
|
112
|
+
# @return [Html2rss::Config] the configuration object.
|
|
113
|
+
def from_hash(config, params: UNSET)
|
|
114
|
+
new(resolve_effective_config(config, params:))
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
##
|
|
118
|
+
# Builds a top-level auto-source feed config for the public shortcut APIs.
|
|
119
|
+
#
|
|
120
|
+
# @param url [String] source page URL
|
|
121
|
+
# @param items_selector [String, nil] optional selector hint for item extraction
|
|
122
|
+
# @param request_controls [Html2rss::Config::RequestControls, nil] explicit request controls to write
|
|
123
|
+
# @return [Hash{Symbol => Object}] feed config hash ready for {from_hash}
|
|
124
|
+
def auto_source_config(url:, items_selector: nil, request_controls: nil)
|
|
125
|
+
config = {
|
|
126
|
+
channel: default_config[:channel].merge(url:),
|
|
127
|
+
auto_source: AutoSource::DEFAULT_CONFIG
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
request_controls ||= RequestControls.new
|
|
131
|
+
request_controls.apply_to(config)
|
|
132
|
+
|
|
133
|
+
config[:selectors] = { items: { selector: items_selector, enhance: true } } if items_selector
|
|
134
|
+
config
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
##
|
|
138
|
+
# Provides a default configuration.
|
|
139
|
+
#
|
|
140
|
+
# @return [Hash{Symbol => Object}] a hash with default configuration values.
|
|
141
|
+
def default_config
|
|
142
|
+
{
|
|
143
|
+
strategy: default_strategy_name,
|
|
144
|
+
request: default_request_config,
|
|
145
|
+
channel: { time_zone: 'UTC' },
|
|
146
|
+
headers: RequestHeaders.browser_defaults,
|
|
147
|
+
stylesheets: Html2rss.defaults.stylesheets || []
|
|
148
|
+
}
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# @return [Symbol] the default feed-level strategy plan (+:auto+ or concrete)
|
|
152
|
+
def default_strategy_name
|
|
153
|
+
Html2rss.defaults.default_strategy || :auto
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
private
|
|
157
|
+
|
|
158
|
+
def default_request_config
|
|
159
|
+
{
|
|
160
|
+
max_redirects: RequestService::Policy::DEFAULTS[:max_redirects],
|
|
161
|
+
max_requests: RequestService::Policy::DEFAULTS[:max_requests],
|
|
162
|
+
total_timeout_seconds: RequestService::Policy::DEFAULTS[:total_timeout_seconds]
|
|
163
|
+
}
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def resolve_effective_config(config, params:)
|
|
167
|
+
cfg = HashUtil.deep_symbolize_keys(config, context: 'config')
|
|
168
|
+
p = parameter_defaults(cfg)
|
|
169
|
+
p.merge!(HashUtil.deep_symbolize_keys(params, context: 'params')) unless params.equal?(UNSET) || params.nil?
|
|
170
|
+
|
|
171
|
+
cfg[:headers] = DynamicParams.call(cfg[:headers], p) if cfg[:headers]
|
|
172
|
+
cfg[:channel] = DynamicParams.call(cfg[:channel], p) if cfg[:channel]
|
|
173
|
+
cfg
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def parameter_defaults(config)
|
|
177
|
+
config.fetch(:parameters, {})
|
|
178
|
+
.filter_map do |name, definition|
|
|
179
|
+
[name, definition[:default]] if definition.is_a?(Hash) && definition.key?(:default)
|
|
180
|
+
end
|
|
181
|
+
.to_h
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def prepare_for_validation(config)
|
|
185
|
+
Config::Preparer.new.call(HashUtil.deep_dup(config))
|
|
186
|
+
end
|
|
187
|
+
end
|
|
17
188
|
|
|
18
189
|
##
|
|
19
190
|
# Initializes the configuration object.
|
|
20
191
|
#
|
|
21
|
-
#
|
|
192
|
+
# Applies default values and validates the configuration.
|
|
22
193
|
#
|
|
23
194
|
# @param config [Hash{Symbol => Object}] the configuration hash.
|
|
24
195
|
# @raise [InvalidConfig] if the configuration fails validation.
|
|
@@ -54,7 +225,7 @@ module Html2rss
|
|
|
54
225
|
end
|
|
55
226
|
|
|
56
227
|
##
|
|
57
|
-
# @return [Html2rss::RequestControls] request controls with provenance
|
|
228
|
+
# @return [Html2rss::Config::RequestControls] request controls with provenance
|
|
58
229
|
attr_reader :request_controls
|
|
59
230
|
|
|
60
231
|
# @return [Hash{String => String}] normalized HTTP headers
|
|
@@ -82,11 +253,10 @@ module Html2rss
|
|
|
82
253
|
class Preparer
|
|
83
254
|
##
|
|
84
255
|
# @param config [Hash{Symbol => Object}] raw config input
|
|
85
|
-
# @return [Hash{Symbol => Object}] config with defaults
|
|
256
|
+
# @return [Hash{Symbol => Object}] config with defaults applied
|
|
86
257
|
def call(config)
|
|
87
258
|
config = config.dup if config.frozen?
|
|
88
259
|
|
|
89
|
-
config = handle_deprecated_channel_attributes(config)
|
|
90
260
|
config = apply_default_config(config)
|
|
91
261
|
config = apply_default_selectors_config(config) if config[:selectors]
|
|
92
262
|
config = apply_default_auto_source_config(config) if config[:auto_source]
|
|
@@ -96,19 +266,6 @@ module Html2rss
|
|
|
96
266
|
|
|
97
267
|
private
|
|
98
268
|
|
|
99
|
-
def handle_deprecated_channel_attributes(config)
|
|
100
|
-
{ strategy: Config.default_strategy_name, headers: {} }.each_pair do |key, default_value|
|
|
101
|
-
if !config[key] && (value = config.dig(:channel, key))
|
|
102
|
-
Log.warn("The `channel.#{key}` key is deprecated. Please move the definition of `#{key}` to the top level.")
|
|
103
|
-
config[key] = value
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
config[key] ||= default_value
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
config
|
|
110
|
-
end
|
|
111
|
-
|
|
112
269
|
def apply_default_config(config)
|
|
113
270
|
HashUtil.deep_merge(Config.default_config, config)
|
|
114
271
|
end
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
##
|
|
5
|
-
# Global
|
|
6
|
-
class
|
|
5
|
+
# Global defaults for the Html2rss gem.
|
|
6
|
+
class Defaults
|
|
7
7
|
# The valid symbol log levels.
|
|
8
8
|
VALID_LOG_LEVELS = %i[debug info warn error fatal unknown].freeze
|
|
9
9
|
|
|
@@ -29,7 +29,7 @@ module Html2rss
|
|
|
29
29
|
attr_reader :stylesheets
|
|
30
30
|
|
|
31
31
|
##
|
|
32
|
-
# Initializes a new
|
|
32
|
+
# Initializes a new Defaults instance.
|
|
33
33
|
def initialize
|
|
34
34
|
@logger_formatter = proc do |severity, datetime, _progname, msg|
|
|
35
35
|
"#{datetime} [#{severity}] #{msg}\n"
|
|
@@ -93,11 +93,13 @@ module Html2rss
|
|
|
93
93
|
end
|
|
94
94
|
|
|
95
95
|
##
|
|
96
|
-
# Sets the default strategy.
|
|
96
|
+
# Sets the default feed-level strategy plan (+:auto+ or a concrete transport strategy).
|
|
97
97
|
#
|
|
98
|
-
#
|
|
99
|
-
#
|
|
100
|
-
# @
|
|
98
|
+
# +:auto+ is not a {RequestService} adapter — it is resolved by {FeedPipeline::StrategyPlan}.
|
|
99
|
+
#
|
|
100
|
+
# @param strategy [Symbol, String, nil] the strategy plan name
|
|
101
|
+
# @return [Symbol, nil] the normalized strategy plan name
|
|
102
|
+
# @raise [ArgumentError] if the strategy plan is unknown
|
|
101
103
|
def default_strategy=(strategy)
|
|
102
104
|
if strategy.nil?
|
|
103
105
|
@default_strategy = nil
|
|
@@ -107,7 +109,7 @@ module Html2rss
|
|
|
107
109
|
end
|
|
108
110
|
|
|
109
111
|
normalized = strategy.to_sym
|
|
110
|
-
raise ArgumentError, "unknown strategy: #{strategy}" unless
|
|
112
|
+
raise ArgumentError, "unknown strategy: #{strategy}" unless FeedPipeline::StrategyPlan.valid?(normalized)
|
|
111
113
|
|
|
112
114
|
@default_strategy = normalized
|
|
113
115
|
end
|
|
@@ -148,9 +150,9 @@ module Html2rss
|
|
|
148
150
|
protected
|
|
149
151
|
|
|
150
152
|
##
|
|
151
|
-
# Copy constructor for duplicating
|
|
153
|
+
# Copy constructor for duplicating defaults.
|
|
152
154
|
#
|
|
153
|
-
# @param other [Html2rss::
|
|
155
|
+
# @param other [Html2rss::Defaults] the original defaults
|
|
154
156
|
# @return [void]
|
|
155
157
|
def initialize_copy(other)
|
|
156
158
|
super
|