html2rss 0.27.1 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/lib/html2rss/auto_source/README.md +4 -1
  3. data/lib/html2rss/auto_source/scraper/native_feed.rb +90 -0
  4. data/lib/html2rss/auto_source/scraper.rb +4 -1
  5. data/lib/html2rss/auto_source.rb +7 -0
  6. data/lib/html2rss/config/auto_source_contract.rb +7 -0
  7. data/lib/html2rss/config.rb +10 -1
  8. data/lib/html2rss/feed_pipeline/README.md +2 -0
  9. data/lib/html2rss/feed_pipeline/auto_fallback.rb +53 -13
  10. data/lib/html2rss/feed_pipeline/runtime_policy.rb +1 -0
  11. data/lib/html2rss/feed_pipeline.rb +24 -8
  12. data/lib/html2rss/feed_resolution/README.md +52 -0
  13. data/lib/html2rss/feed_resolution/candidate_generator.rb +139 -0
  14. data/lib/html2rss/feed_resolution/diag.rb +32 -0
  15. data/lib/html2rss/feed_resolution/options.rb +31 -0
  16. data/lib/html2rss/feed_resolution/policy.rb +47 -0
  17. data/lib/html2rss/feed_resolution/probe.rb +59 -0
  18. data/lib/html2rss/feed_resolution/scorer.rb +49 -0
  19. data/lib/html2rss/feed_resolution.rb +274 -0
  20. data/lib/html2rss/mcp/inspect.rb +18 -121
  21. data/lib/html2rss/mcp/outcome.rb +4 -2
  22. data/lib/html2rss/page_recon.rb +243 -0
  23. data/lib/html2rss/request_service/botasaurus_contract.rb +24 -47
  24. data/lib/html2rss/request_service/botasaurus_strategy.rb +34 -12
  25. data/lib/html2rss/request_service/response.rb +26 -2
  26. data/lib/html2rss/request_service.rb +13 -1
  27. data/lib/html2rss/request_session.rb +5 -2
  28. data/lib/html2rss/scrape_target.rb +24 -0
  29. data/lib/html2rss/status.rb +49 -22
  30. data/lib/html2rss/surface_category.rb +63 -0
  31. data/lib/html2rss/syndication/README.md +21 -0
  32. data/lib/html2rss/syndication/candidate_catalog.rb +26 -0
  33. data/lib/html2rss/syndication/discovery.rb +217 -0
  34. data/lib/html2rss/syndication/parser.rb +137 -0
  35. data/lib/html2rss/syndication.rb +10 -0
  36. data/lib/html2rss/url.rb +22 -0
  37. data/lib/html2rss/version.rb +1 -1
  38. data/schema/html2rss-config.schema.json +39 -1
  39. metadata +19 -2
data/schema/html2rss-config.schema.json CHANGED
@@ -157,9 +157,40 @@
157
157
  },
158
158
  "exclusiveMinimum": 0
159
159
  },
160
+ "entry_resolution": {
161
+ "type": "object",
162
+ "properties": {
163
+ "enabled": {
164
+ "type": "boolean",
165
+ "not": {
166
+ "type": "null"
167
+ }
168
+ },
169
+ "max_probes": {
170
+ "type": "integer",
171
+ "not": {
172
+ "type": "null"
173
+ },
174
+ "exclusiveMinimum": 0
175
+ }
176
+ },
177
+ "required": []
178
+ },
160
179
  "scraper": {
161
180
  "type": "object",
162
181
  "properties": {
182
+ "native_feed": {
183
+ "type": "object",
184
+ "properties": {
185
+ "enabled": {
186
+ "type": "boolean",
187
+ "not": {
188
+ "type": "null"
189
+ }
190
+ }
191
+ },
192
+ "required": []
193
+ },
163
194
  "wordpress_api": {
164
195
  "type": "object",
165
196
  "properties": {
@@ -338,7 +369,14 @@
338
369
  "required": [],
339
370
  "default": {
340
371
  "limit": 25,
372
+ "entry_resolution": {
373
+ "enabled": true,
374
+ "max_probes": 5
375
+ },
341
376
  "scraper": {
377
+ "native_feed": {
378
+ "enabled": true
379
+ },
342
380
  "wordpress_api": {
343
381
  "enabled": true
344
382
  },
@@ -698,7 +736,7 @@
698
736
  "type": "null"
699
737
  },
700
738
  "minimum": 1,
701
- "maximum": 20
739
+ "maximum": 30
702
740
  },
703
741
  "scroll": {
704
742
  "type": "boolean",
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: html2rss
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.27.1
4
+ version: 0.28.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Gil Desmarais
@@ -331,6 +331,7 @@ files:
331
331
  - lib/html2rss/auto_source/scraper/meta_oembed.rb
332
332
  - lib/html2rss/auto_source/scraper/microdata.rb
333
333
  - lib/html2rss/auto_source/scraper/microformats2.rb
334
+ - lib/html2rss/auto_source/scraper/native_feed.rb
334
335
  - lib/html2rss/auto_source/scraper/schema.rb
335
336
  - lib/html2rss/auto_source/scraper/schema/category_extractor.rb
336
337
  - lib/html2rss/auto_source/scraper/schema/item_list.rb
@@ -376,6 +377,14 @@ files:
376
377
  - lib/html2rss/feed_pipeline/auto_fallback.rb
377
378
  - lib/html2rss/feed_pipeline/runtime_policy.rb
378
379
  - lib/html2rss/feed_pipeline/strategy_plan.rb
380
+ - lib/html2rss/feed_resolution.rb
381
+ - lib/html2rss/feed_resolution/README.md
382
+ - lib/html2rss/feed_resolution/candidate_generator.rb
383
+ - lib/html2rss/feed_resolution/diag.rb
384
+ - lib/html2rss/feed_resolution/options.rb
385
+ - lib/html2rss/feed_resolution/policy.rb
386
+ - lib/html2rss/feed_resolution/probe.rb
387
+ - lib/html2rss/feed_resolution/scorer.rb
379
388
  - lib/html2rss/feed_result.rb
380
389
  - lib/html2rss/hash_util.rb
381
390
  - lib/html2rss/html.rb
@@ -416,6 +425,7 @@ files:
416
425
  - lib/html2rss/mcp/inspect.rb
417
426
  - lib/html2rss/mcp/outcome.rb
418
427
  - lib/html2rss/mcp/server.rb
428
+ - lib/html2rss/page_recon.rb
419
429
  - lib/html2rss/request_service.rb
420
430
  - lib/html2rss/request_service/blocked_surface.rb
421
431
  - lib/html2rss/request_service/botasaurus_contract.rb
@@ -446,6 +456,7 @@ files:
446
456
  - lib/html2rss/scoring/observation.rb
447
457
  - lib/html2rss/scoring/ranked_segment.rb
448
458
  - lib/html2rss/scoring/score.rb
459
+ - lib/html2rss/scrape_target.rb
449
460
  - lib/html2rss/selectors.rb
450
461
  - lib/html2rss/selectors/extractors.rb
451
462
  - lib/html2rss/selectors/extractors/attribute.rb
@@ -477,6 +488,12 @@ files:
477
488
  - lib/html2rss/sst/tags.rb
478
489
  - lib/html2rss/sst/text.rb
479
490
  - lib/html2rss/status.rb
491
+ - lib/html2rss/surface_category.rb
492
+ - lib/html2rss/syndication.rb
493
+ - lib/html2rss/syndication/README.md
494
+ - lib/html2rss/syndication/candidate_catalog.rb
495
+ - lib/html2rss/syndication/discovery.rb
496
+ - lib/html2rss/syndication/parser.rb
480
497
  - lib/html2rss/url.rb
481
498
  - lib/html2rss/version.rb
482
499
  - lib/tasks/config_schema.rake
@@ -486,7 +503,7 @@ licenses:
486
503
  - MIT
487
504
  metadata:
488
505
  allowed_push_host: https://rubygems.org
489
- changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.27.1
506
+ changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.28.0
490
507
  rubygems_mfa_required: 'true'
491
508
  rdoc_options: []
492
509
  require_paths: