html2rss 0.34.0 → 0.34.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/html2rss +4 -0
- data/lib/html2rss/request_service/response.rb +24 -1
- data/lib/html2rss/selectors.rb +1 -1
- data/lib/html2rss/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e91a63f8267b667a0bc1fc7ea2a17419b4386b7afa30d081c1bda9dc6f5ccd5f
|
|
4
|
+
data.tar.gz: fb785c14e1e6463d00f77b0cbc3ecf16689e3f326f5cda333e183f8c4db5c61c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1cfd939a70acd9a4ce61c7b2ce4f06e3fbac729baec3e34c795c5b6d2f042d40f195932c24bea776b83da98f12b8020d4cb383814d42be6ba45c2ba133611773
|
|
7
|
+
data.tar.gz: c3b43b4bfffa6ad0e1873a76d0aef74935b4512a4d8262eb4b156207f634f39942c0df265776a35a1d3e8218f1c002a336fb6d16128fbfdebb9139a216cf11c5
|
data/exe/html2rss
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
# MCP clients and cron often spawn this process without LANG, which leaves Ruby
|
|
5
|
+
# on US-ASCII. Bundled data (public_suffix's list) and configs are UTF-8.
|
|
6
|
+
Encoding.default_external = Encoding::UTF_8
|
|
7
|
+
|
|
4
8
|
require 'html2rss'
|
|
5
9
|
|
|
6
10
|
Html2rss::CLI.start(ARGV)
|
|
@@ -24,7 +24,12 @@ module Html2rss
|
|
|
24
24
|
CHARSET_PARAMETER = /charset\s*=\s*["']?([\w.:-]+)/i
|
|
25
25
|
# Bytes scanned for a meta charset hint (HTML spec looks at the first 1024).
|
|
26
26
|
META_CHARSET_BYTES = 2048
|
|
27
|
-
|
|
27
|
+
# The element that sets the document base URL: the first +<base>+ with an +href+ in tree order, unless the
|
|
28
|
+
# parser leaves it inert (inside +template+, +noscript+, +iframe+, +textarea+ or +title+) or it is not an
|
|
29
|
+
# HTML +base+ (inside +svg+).
|
|
30
|
+
BASE_ELEMENT = '//base[@href][not(ancestor::template or ancestor::noscript or ancestor::iframe or ' \
|
|
31
|
+
'ancestor::textarea or ancestor::title or ancestor::svg)]'
|
|
32
|
+
private_constant :BASE_ELEMENT, :CHARSET_PARAMETER, :META_CHARSET_BYTES, :FEED_CT_MARKERS, :FEED_BODY_MARKERS
|
|
28
33
|
|
|
29
34
|
##
|
|
30
35
|
# @param body [String] the body of the response
|
|
@@ -61,6 +66,18 @@ module Html2rss
|
|
|
61
66
|
# @return [Html2rss::Url] the URL of the response
|
|
62
67
|
attr_reader :url
|
|
63
68
|
|
|
69
|
+
##
|
|
70
|
+
# The document base URL (WHATWG) that relative references in the body resolve against: the first
|
|
71
|
+
# +<base href>+ of an HTML body in tree order (one inside +template+, +noscript+, +iframe+, +textarea+,
|
|
72
|
+
# +title+ or +svg+ does not count) resolved against {#url}, when that is http(s); otherwise {#url}
|
|
73
|
+
# (no +<base>+, an empty +href+, a +javascript:+ or +data:+ one, or a non-HTML body). Only {Selectors} reads
|
|
74
|
+
# it, to resolve item links and enclosures; auto-source, pagination, discovery and every request keep {#url}.
|
|
75
|
+
#
|
|
76
|
+
# @return [Html2rss::Url] the URL a selector config's relative references resolve against
|
|
77
|
+
def base_url
|
|
78
|
+
@base_url ||= document_base_url
|
|
79
|
+
end
|
|
80
|
+
|
|
64
81
|
# @return [Hash] allowlisted upstream transport telemetry
|
|
65
82
|
attr_reader :transport_meta
|
|
66
83
|
|
|
@@ -116,6 +133,12 @@ module Html2rss
|
|
|
116
133
|
|
|
117
134
|
private
|
|
118
135
|
|
|
136
|
+
# @return [Html2rss::Url] the first usable +<base href>+ resolved against {#url}, else {#url}
|
|
137
|
+
def document_base_url
|
|
138
|
+
href = parsed_body.at_xpath(BASE_ELEMENT)&.[]('href') if html_response?
|
|
139
|
+
Url.resolve_web(href, url) || url
|
|
140
|
+
end
|
|
141
|
+
|
|
119
142
|
def freeze_or(value, default) = value.nil? || value.empty? ? default : value.freeze
|
|
120
143
|
|
|
121
144
|
# @param name [String] canonical header name
|
data/lib/html2rss/selectors.rb
CHANGED
|
@@ -55,7 +55,7 @@ module Html2rss
|
|
|
55
55
|
# @param time_zone [String] Time zone string used for date parsing.
|
|
56
56
|
def initialize(response, selectors:, time_zone:)
|
|
57
57
|
@response = response
|
|
58
|
-
@url = response.
|
|
58
|
+
@url = response.base_url
|
|
59
59
|
@selectors = selectors
|
|
60
60
|
@time_zone = time_zone
|
|
61
61
|
@rss_item_attributes = @selectors.keys.select { SELECTABLE_SELECTOR_KEYS.include?(_1) }
|
data/lib/html2rss/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: html2rss
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.34.
|
|
4
|
+
version: 0.34.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gil Desmarais
|
|
@@ -507,7 +507,7 @@ licenses:
|
|
|
507
507
|
- MIT
|
|
508
508
|
metadata:
|
|
509
509
|
allowed_push_host: https://rubygems.org
|
|
510
|
-
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.34.
|
|
510
|
+
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.34.1
|
|
511
511
|
rubygems_mfa_required: 'true'
|
|
512
512
|
rdoc_options: []
|
|
513
513
|
require_paths:
|