html2rss 0.33.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/lib/html2rss/article/deduplicator.rb +5 -2
- data/lib/html2rss/article/enclosure.rb +185 -21
- data/lib/html2rss/article/title.rb +203 -0
- data/lib/html2rss/article.rb +79 -13
- data/lib/html2rss/auto_source/README.md +16 -8
- data/lib/html2rss/auto_source/cleanup.rb +126 -50
- data/lib/html2rss/auto_source/corroboration.rb +167 -0
- data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
- data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
- data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
- data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
- data/lib/html2rss/auto_source/page_links.rb +164 -0
- data/lib/html2rss/auto_source/scraper/html.rb +11 -39
- data/lib/html2rss/auto_source/scraper/json_document.rb +1 -3
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +119 -42
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +5 -16
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +2 -10
- data/lib/html2rss/auto_source/scraper/json_state.rb +2 -3
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +6 -14
- data/lib/html2rss/auto_source/scraper/microdata.rb +6 -4
- data/lib/html2rss/auto_source/scraper/microformats2.rb +1 -3
- data/lib/html2rss/auto_source/scraper/native_feed.rb +27 -17
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +68 -27
- data/lib/html2rss/auto_source/scraper/schema.rb +5 -8
- data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +1 -5
- data/lib/html2rss/auto_source/scraper/sitemap.rb +4 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -3
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -3
- data/lib/html2rss/auto_source/scraper.rb +10 -2
- data/lib/html2rss/auto_source/segment.rb +19 -5
- data/lib/html2rss/auto_source/segmenter/cluster.rb +13 -11
- data/lib/html2rss/auto_source/segmenter/list.rb +13 -5
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
- data/lib/html2rss/auto_source/segmenter/semantic.rb +17 -6
- data/lib/html2rss/auto_source/segmenter.rb +4 -8
- data/lib/html2rss/auto_source.rb +81 -26
- data/lib/html2rss/batch.rb +2 -0
- data/lib/html2rss/capture/README.md +5 -2
- data/lib/html2rss/capture.rb +44 -32
- data/lib/html2rss/channel.rb +30 -12
- data/lib/html2rss/cli.rb +22 -11
- data/lib/html2rss/config/auto_source_contract.rb +52 -45
- data/lib/html2rss/config/issue_mapper.rb +4 -1
- data/lib/html2rss/config/request_controls.rb +6 -3
- data/lib/html2rss/config/schema.rb +15 -5
- data/lib/html2rss/config/selectors_validator.rb +2 -2
- data/lib/html2rss/config/validation_issue.rb +4 -1
- data/lib/html2rss/config/validator.rb +11 -0
- data/lib/html2rss/config.rb +53 -27
- data/lib/html2rss/contract_error.rb +9 -0
- data/lib/html2rss/defaults.rb +21 -23
- data/lib/html2rss/doctor/botasaurus.rb +7 -3
- data/lib/html2rss/error.rb +10 -0
- data/lib/html2rss/error_kind.rb +39 -0
- data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
- data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
- data/lib/html2rss/feed_builder/json_feed.rb +2 -2
- data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
- data/lib/html2rss/feed_builder/rss.rb +49 -6
- data/lib/html2rss/feed_builder.rb +2 -1
- data/lib/html2rss/feed_pipeline/README.md +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +13 -31
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
- data/lib/html2rss/feed_pipeline.rb +40 -27
- data/lib/html2rss/feed_resolution/README.md +1 -1
- data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
- data/lib/html2rss/feed_resolution/policy.rb +1 -1
- data/lib/html2rss/feed_resolution/probe.rb +3 -1
- data/lib/html2rss/feed_resolution.rb +11 -12
- data/lib/html2rss/feed_result.rb +9 -5
- data/lib/html2rss/hash_util.rb +3 -3
- data/lib/html2rss/html/article_extractor/category_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -31
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +6 -9
- data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
- data/lib/html2rss/html/article_extractor.rb +19 -47
- data/lib/html2rss/html/article_rules/category.rb +3 -3
- data/lib/html2rss/html/article_rules/description.rb +6 -15
- data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
- data/lib/html2rss/html/article_rules/image.rb +30 -16
- data/lib/html2rss/html/feed_link.rb +7 -9
- data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
- data/lib/html2rss/html/navigator.rb +39 -9
- data/lib/html2rss/html/probe.rb +4 -0
- data/lib/html2rss/html/rendering/description_builder.rb +46 -32
- data/lib/html2rss/html/rendering.rb +2 -9
- data/lib/html2rss/html/sst_article_extractor.rb +55 -145
- data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
- data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
- data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
- data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
- data/lib/html2rss/html/title_pieces.rb +169 -0
- data/lib/html2rss/html/visibility.rb +56 -0
- data/lib/html2rss/html.rb +2 -0
- data/lib/html2rss/link_destination/destination_facts.rb +17 -1
- data/lib/html2rss/link_destination/noise_policy.rb +11 -10
- data/lib/html2rss/link_destination/path_classifier.rb +100 -9
- data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
- data/lib/html2rss/link_destination/sponsorship.rb +184 -0
- data/lib/html2rss/link_destination/text_classifier.rb +104 -8
- data/lib/html2rss/mcp/README.md +13 -1
- data/lib/html2rss/mcp/config_argument.rb +5 -2
- data/lib/html2rss/mcp/contract.rb +13 -4
- data/lib/html2rss/mcp/outcome.rb +14 -12
- data/lib/html2rss/mcp/runtime.rb +39 -2
- data/lib/html2rss/mcp/server.rb +13 -4
- data/lib/html2rss/page_recon/README.md +1 -1
- data/lib/html2rss/page_recon/diagnostics.rb +2 -0
- data/lib/html2rss/recon.rb +10 -12
- data/lib/html2rss/request_service/blocked_surface.rb +2 -1
- data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
- data/lib/html2rss/request_service/botasaurus_strategy.rb +42 -10
- data/lib/html2rss/request_service/context.rb +15 -14
- data/lib/html2rss/request_service/cookie_jar.rb +89 -0
- data/lib/html2rss/request_service/cookie_store.rb +79 -0
- data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
- data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +2 -2
- data/lib/html2rss/request_service/policy.rb +48 -11
- data/lib/html2rss/request_service/replay_breaker.rb +56 -0
- data/lib/html2rss/request_service/response.rb +30 -13
- data/lib/html2rss/request_service/response_cache.rb +109 -0
- data/lib/html2rss/request_service/response_caching.rb +95 -0
- data/lib/html2rss/request_service/strategy.rb +5 -11
- data/lib/html2rss/request_service.rb +71 -12
- data/lib/html2rss/request_session/pager/base.rb +0 -7
- data/lib/html2rss/request_session/pager.rb +1 -1
- data/lib/html2rss/request_session.rb +6 -2
- data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
- data/lib/html2rss/scoring/container_assessor.rb +12 -13
- data/lib/html2rss/scoring/engine.rb +6 -6
- data/lib/html2rss/scoring/observation.rb +2 -2
- data/lib/html2rss/selector_candidates.rb +19 -8
- data/lib/html2rss/selectors/README.md +1 -1
- data/lib/html2rss/selectors/option_spec.rb +4 -1
- data/lib/html2rss/selectors/post_processors/base.rb +0 -8
- data/lib/html2rss/selectors/post_processors/gsub.rb +7 -7
- data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +3 -3
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
- data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
- data/lib/html2rss/selectors.rb +53 -12
- data/lib/html2rss/sst/attrs.rb +2 -2
- data/lib/html2rss/sst/document.rb +3 -3
- data/lib/html2rss/sst/index.rb +30 -11
- data/lib/html2rss/sst/node.rb +33 -26
- data/lib/html2rss/sst/normalizer.rb +31 -19
- data/lib/html2rss/sst/tags.rb +113 -2
- data/lib/html2rss/sst/text.rb +69 -29
- data/lib/html2rss/sst.rb +2 -0
- data/lib/html2rss/status.rb +6 -3
- data/lib/html2rss/surface_category.rb +1 -1
- data/lib/html2rss/syndication/README.md +12 -6
- data/lib/html2rss/syndication/alternate_links.rb +1 -1
- data/lib/html2rss/syndication/channel_facts.rb +87 -0
- data/lib/html2rss/syndication/discovery.rb +82 -28
- data/lib/html2rss/syndication/feed.rb +20 -0
- data/lib/html2rss/syndication/item_media.rb +103 -0
- data/lib/html2rss/syndication/parser.rb +75 -51
- data/lib/html2rss/test/enhance_audit.rb +2 -2
- data/lib/html2rss/test/item_sample.rb +43 -0
- data/lib/html2rss/test.rb +42 -14
- data/lib/html2rss/url.rb +115 -14
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss/word_count.rb +53 -0
- data/lib/html2rss.rb +17 -8
- data/schema/html2rss-config.schema.json +56 -41
- metadata +31 -4
- data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
- data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: html2rss
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.34.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gil Desmarais
|
|
@@ -270,9 +270,16 @@ files:
|
|
|
270
270
|
- lib/html2rss/article.rb
|
|
271
271
|
- lib/html2rss/article/deduplicator.rb
|
|
272
272
|
- lib/html2rss/article/enclosure.rb
|
|
273
|
+
- lib/html2rss/article/title.rb
|
|
273
274
|
- lib/html2rss/auto_source.rb
|
|
274
275
|
- lib/html2rss/auto_source/README.md
|
|
275
276
|
- lib/html2rss/auto_source/cleanup.rb
|
|
277
|
+
- lib/html2rss/auto_source/corroboration.rb
|
|
278
|
+
- lib/html2rss/auto_source/page_links.rb
|
|
279
|
+
- lib/html2rss/auto_source/page_links/anchor_claims.rb
|
|
280
|
+
- lib/html2rss/auto_source/page_links/card_units.rb
|
|
281
|
+
- lib/html2rss/auto_source/page_links/label_binding.rb
|
|
282
|
+
- lib/html2rss/auto_source/page_links/sponsored_cards.rb
|
|
276
283
|
- lib/html2rss/auto_source/scraper.rb
|
|
277
284
|
- lib/html2rss/auto_source/scraper/html.rb
|
|
278
285
|
- lib/html2rss/auto_source/scraper/json_document.rb
|
|
@@ -324,10 +331,12 @@ files:
|
|
|
324
331
|
- lib/html2rss/config/validation_issue.rb
|
|
325
332
|
- lib/html2rss/config/validation_report.rb
|
|
326
333
|
- lib/html2rss/config/validator.rb
|
|
334
|
+
- lib/html2rss/contract_error.rb
|
|
327
335
|
- lib/html2rss/defaults.rb
|
|
328
336
|
- lib/html2rss/doctor.rb
|
|
329
337
|
- lib/html2rss/doctor/botasaurus.rb
|
|
330
338
|
- lib/html2rss/error.rb
|
|
339
|
+
- lib/html2rss/error_kind.rb
|
|
331
340
|
- lib/html2rss/feed_builder.rb
|
|
332
341
|
- lib/html2rss/feed_builder/item_presentation.rb
|
|
333
342
|
- lib/html2rss/feed_builder/json_feed.rb
|
|
@@ -373,14 +382,20 @@ files:
|
|
|
373
382
|
- lib/html2rss/html/rendering/description_builder.rb
|
|
374
383
|
- lib/html2rss/html/rendering/escaped_attributes.rb
|
|
375
384
|
- lib/html2rss/html/rendering/image_renderer.rb
|
|
376
|
-
- lib/html2rss/html/rendering/media_renderer.rb
|
|
377
|
-
- lib/html2rss/html/rendering/pdf_renderer.rb
|
|
378
385
|
- lib/html2rss/html/rendering/video_renderer.rb
|
|
379
386
|
- lib/html2rss/html/sst_article_extractor.rb
|
|
387
|
+
- lib/html2rss/html/title_pieces.rb
|
|
388
|
+
- lib/html2rss/html/title_pieces/kicker.rb
|
|
389
|
+
- lib/html2rss/html/title_pieces/nodes.rb
|
|
390
|
+
- lib/html2rss/html/title_pieces/nokogiri_nodes.rb
|
|
391
|
+
- lib/html2rss/html/title_pieces/sst_nodes.rb
|
|
392
|
+
- lib/html2rss/html/visibility.rb
|
|
380
393
|
- lib/html2rss/link_destination.rb
|
|
381
394
|
- lib/html2rss/link_destination/destination_facts.rb
|
|
382
395
|
- lib/html2rss/link_destination/noise_policy.rb
|
|
383
396
|
- lib/html2rss/link_destination/path_classifier.rb
|
|
397
|
+
- lib/html2rss/link_destination/sponsorship.rb
|
|
398
|
+
- lib/html2rss/link_destination/sponsorship/label_text.rb
|
|
384
399
|
- lib/html2rss/link_destination/text_classifier.rb
|
|
385
400
|
- lib/html2rss/mcp.rb
|
|
386
401
|
- lib/html2rss/mcp/README.md
|
|
@@ -403,11 +418,17 @@ files:
|
|
|
403
418
|
- lib/html2rss/request_service/budget.rb
|
|
404
419
|
- lib/html2rss/request_service/compressed_body.rb
|
|
405
420
|
- lib/html2rss/request_service/context.rb
|
|
421
|
+
- lib/html2rss/request_service/cookie_jar.rb
|
|
422
|
+
- lib/html2rss/request_service/cookie_store.rb
|
|
423
|
+
- lib/html2rss/request_service/feed_advertisement.rb
|
|
406
424
|
- lib/html2rss/request_service/httpx_strategy.rb
|
|
407
425
|
- lib/html2rss/request_service/local_file_strategy.rb
|
|
408
426
|
- lib/html2rss/request_service/network_guard.rb
|
|
409
427
|
- lib/html2rss/request_service/policy.rb
|
|
428
|
+
- lib/html2rss/request_service/replay_breaker.rb
|
|
410
429
|
- lib/html2rss/request_service/response.rb
|
|
430
|
+
- lib/html2rss/request_service/response_cache.rb
|
|
431
|
+
- lib/html2rss/request_service/response_caching.rb
|
|
411
432
|
- lib/html2rss/request_service/response_guard.rb
|
|
412
433
|
- lib/html2rss/request_service/strategy.rb
|
|
413
434
|
- lib/html2rss/request_session.rb
|
|
@@ -442,6 +463,7 @@ files:
|
|
|
442
463
|
- lib/html2rss/selectors/post_processors.rb
|
|
443
464
|
- lib/html2rss/selectors/post_processors/base.rb
|
|
444
465
|
- lib/html2rss/selectors/post_processors/gsub.rb
|
|
466
|
+
- lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb
|
|
445
467
|
- lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
|
|
446
468
|
- lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb
|
|
447
469
|
- lib/html2rss/selectors/post_processors/markdown_to_html.rb
|
|
@@ -466,13 +488,18 @@ files:
|
|
|
466
488
|
- lib/html2rss/syndication/README.md
|
|
467
489
|
- lib/html2rss/syndication/alternate_links.rb
|
|
468
490
|
- lib/html2rss/syndication/candidate_catalog.rb
|
|
491
|
+
- lib/html2rss/syndication/channel_facts.rb
|
|
469
492
|
- lib/html2rss/syndication/discovery.rb
|
|
493
|
+
- lib/html2rss/syndication/feed.rb
|
|
494
|
+
- lib/html2rss/syndication/item_media.rb
|
|
470
495
|
- lib/html2rss/syndication/parser.rb
|
|
471
496
|
- lib/html2rss/test.rb
|
|
472
497
|
- lib/html2rss/test/enhance_audit.rb
|
|
498
|
+
- lib/html2rss/test/item_sample.rb
|
|
473
499
|
- lib/html2rss/test/policy.rb
|
|
474
500
|
- lib/html2rss/url.rb
|
|
475
501
|
- lib/html2rss/version.rb
|
|
502
|
+
- lib/html2rss/word_count.rb
|
|
476
503
|
- lib/tasks/config_schema.rake
|
|
477
504
|
- schema/html2rss-config.schema.json
|
|
478
505
|
homepage: https://github.com/html2rss/html2rss
|
|
@@ -480,7 +507,7 @@ licenses:
|
|
|
480
507
|
- MIT
|
|
481
508
|
metadata:
|
|
482
509
|
allowed_push_host: https://rubygems.org
|
|
483
|
-
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.
|
|
510
|
+
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.34.0
|
|
484
511
|
rubygems_mfa_required: 'true'
|
|
485
512
|
rdoc_options: []
|
|
486
513
|
require_paths:
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Html2rss
|
|
4
|
-
module Html
|
|
5
|
-
module Rendering
|
|
6
|
-
# Factory: picks the appropriate renderer for a given enclosure or fallback image.
|
|
7
|
-
class MediaRenderer
|
|
8
|
-
# @param enclosure [Html2rss::Article::Enclosure, nil]
|
|
9
|
-
# @param image [String, Html2rss::Url, nil] Fallback image URL
|
|
10
|
-
# @param title [String]
|
|
11
|
-
# @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
|
|
12
|
-
def self.for(enclosure:, image:, title:)
|
|
13
|
-
return ImageRenderer.new(url: image, title:) if enclosure.nil? && image
|
|
14
|
-
return nil unless enclosure
|
|
15
|
-
|
|
16
|
-
create_renderer_for_type(enclosure.type, url: enclosure.url, title:)
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
# @private
|
|
20
|
-
# @param type [String, nil] enclosure MIME type
|
|
21
|
-
# @param url [String, Html2rss::Url] enclosure URL
|
|
22
|
-
# @param title [String, nil] title used by image renderer
|
|
23
|
-
# @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
|
|
24
|
-
def self.create_renderer_for_type(type, url:, title:)
|
|
25
|
-
case type
|
|
26
|
-
when %r{^image/}
|
|
27
|
-
ImageRenderer.new(url:, title:)
|
|
28
|
-
when %r{^video/}
|
|
29
|
-
VideoRenderer.new(url:, type:)
|
|
30
|
-
when %r{^audio/}
|
|
31
|
-
AudioRenderer.new(url:, type:)
|
|
32
|
-
when 'application/pdf'
|
|
33
|
-
PdfRenderer.new(url:)
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
end
|
|
38
|
-
end
|
|
39
|
-
end
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Html2rss
|
|
4
|
-
module Html
|
|
5
|
-
module Rendering
|
|
6
|
-
# Renders an HTML <iframe> for PDF documents.
|
|
7
|
-
class PdfRenderer
|
|
8
|
-
include EscapedAttributes
|
|
9
|
-
|
|
10
|
-
# @param url [String, Html2rss::Url] PDF URL rendered in the iframe
|
|
11
|
-
def initialize(url:)
|
|
12
|
-
@url = url
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
# @return [String] HTML iframe snippet for PDF rendering
|
|
16
|
-
def to_html
|
|
17
|
-
attributes = [
|
|
18
|
-
%(src="#{escaped_url}"),
|
|
19
|
-
'width="100%"',
|
|
20
|
-
'height="75vh"',
|
|
21
|
-
'sandbox=""',
|
|
22
|
-
'referrerpolicy="no-referrer"',
|
|
23
|
-
'loading="lazy"'
|
|
24
|
-
]
|
|
25
|
-
"<iframe #{attributes.join(' ')}></iframe>"
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
end
|