fasthtmlparser 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,164 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ # @api private HTML5 character reference decoder (WHATWG "character reference state"): all 2125 named references,
5
+ # the legacy names without ';', numeric references with or without ';', and the spec replacements (0, surrogates,
6
+ # > U+10FFFF -> U+FFFD; 0x80-0x9F -> Windows-1252).
7
+ module Decode
8
+ # Replacements for numeric references 0x80..0x9F; 0 = keep the code point as is.
9
+ WINDOWS_1252 = [
10
+ 0x20ac, 0, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017d, 0,
11
+ 0, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0, 0x017e,
12
+ 0x0178
13
+ ].freeze
14
+
15
+ MAP = Entities::MAP
16
+ LEGACY = Entities::LEGACY
17
+ MAX_NAME = Entities::MAX_NAME
18
+ MAX_LEGACY_NAME = Entities::MAX_LEGACY_NAME
19
+ AMP = "&"
20
+ CR = "\r"
21
+
22
+ module_function
23
+
24
+ def alnum?(c)
25
+ c && ((c >= 48 && c <= 57) || ((c | 0x20) >= 97 && (c | 0x20) <= 122))
26
+ end
27
+
28
+ # Whether a value read from the page differs from its source: it has a character reference or a CR.
29
+ def needs_decoding?(s)
30
+ s.include?(AMP) || s.include?(CR)
31
+ end
32
+
33
+ # HTML input stream preprocessing: CR LF and a lone CR become LF.
34
+ def normalize_newlines(s)
35
+ return s unless s.include?(CR)
36
+
37
+ s.gsub("\r\n", "\n").tr("\r", "\n")
38
+ end
39
+
40
+ # Decodes the character references in s; CR LF and a lone CR become LF. With in_attribute, a legacy name without
41
+ # ';' followed by '=' or an alphanumeric is left as is (so "?a=1&copy=2" stays intact). Returns a new string or s.
42
+ def decode(s, in_attribute)
43
+ # Line breaks first, as a browser reads the page; a CR written as "&#13;" is decoded after that and stays.
44
+ s = normalize_newlines(s) if s.include?(CR)
45
+ amp = s.index(AMP)
46
+ return s if amp.nil?
47
+
48
+ n = s.bytesize
49
+ out = String.new(capacity: n, encoding: Encoding::BINARY)
50
+ pos = 0
51
+ while amp
52
+ out << s.byteslice(pos, amp - pos) if amp > pos
53
+ consumed = 0
54
+ text = nil
55
+ if amp + 1 < n
56
+ text, consumed = reference(s, amp + 1, in_attribute)
57
+ end
58
+ if text.nil?
59
+ out << AMP # not a reference: keep '&'
60
+ pos = amp + 1
61
+ else
62
+ out << text
63
+ pos = amp + 1 + consumed
64
+ end
65
+ amp = pos < n ? s.index(AMP, pos) : nil
66
+ end
67
+ out << s.byteslice(pos, n - pos) if pos < n
68
+ out
69
+ end
70
+
71
+ # Decodes one character reference; s[at] is the char right after '&'. Returns [text, bytes consumed after '&'],
72
+ # or [nil, 0] when there is no reference.
73
+ def decode_reference(s, at)
74
+ return [nil, 0] if at >= s.bytesize
75
+
76
+ reference(s, at, false)
77
+ end
78
+
79
+ def reference(s, at, in_attribute)
80
+ if s.getbyte(at) == 35
81
+ cp, consumed = match_numeric(s, at)
82
+ return [nil, 0] if cp < 0
83
+
84
+ return [Chars.utf8(cp), consumed]
85
+ end
86
+ match_named(s, at, in_attribute)
87
+ end
88
+
89
+ # s[at] is '#'. Returns [code point (-1: not a reference), bytes consumed].
90
+ def match_numeric(s, at)
91
+ n = s.bytesize
92
+ i = at + 1
93
+ c = i < n ? s.getbyte(i) : nil
94
+ hex = c == 120 || c == 88
95
+ i += 1 if hex
96
+ digits = i
97
+ value = 0
98
+ if hex
99
+ while i < n
100
+ c = s.getbyte(i)
101
+ if c >= 48 && c <= 57
102
+ d = c - 48
103
+ else
104
+ l = c | 0x20
105
+ break if l < 97 || l > 102
106
+
107
+ d = l - 87
108
+ end
109
+ value = value * 16 + d
110
+ value = 0x110000 if value > 0x10ffff # saturate on long digit runs
111
+ i += 1
112
+ end
113
+ else
114
+ while i < n
115
+ c = s.getbyte(i)
116
+ break if c < 48 || c > 57
117
+
118
+ value = value * 10 + c - 48
119
+ value = 0x110000 if value > 0x10ffff
120
+ i += 1
121
+ end
122
+ end
123
+ return [-1, 0] if i == digits # "&#" / "&#x" without digits is plain text
124
+
125
+ i += 1 if i < n && s.getbyte(i) == 59
126
+ if value == 0 || value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff)
127
+ value = 0xfffd
128
+ elsif value >= 0x80 && value <= 0x9f
129
+ mapped = WINDOWS_1252[value - 0x80]
130
+ value = mapped if mapped != 0
131
+ end
132
+ [value, i - at]
133
+ end
134
+
135
+ # s[at] is the char right after '&'. Returns [decoded text (nil: not a reference), bytes consumed].
136
+ def match_named(s, at, in_attribute)
137
+ len = s.bytesize
138
+ n = 0
139
+ limit = MAX_NAME + 1
140
+ n += 1 while n < limit && at + n < len && alnum?(s.getbyte(at + n))
141
+ return [nil, 0] if n < 2
142
+
143
+ # Full match: "&name;".
144
+ if at + n < len && s.getbyte(at + n) == 59
145
+ v = MAP[s.byteslice(at, n)]
146
+ return [v, n + 1] if v
147
+ end
148
+ # Longest legacy prefix without ';': "&copy2024" -> "(c)2024", "&notit;" -> "(not)it;".
149
+ k = n < MAX_LEGACY_NAME ? n : MAX_LEGACY_NAME
150
+ while k >= 2
151
+ name = s.byteslice(at, k)
152
+ if LEGACY[name]
153
+ if in_attribute && at + k < len
154
+ c = s.getbyte(at + k)
155
+ break if c == 61 || alnum?(c)
156
+ end
157
+ return [MAP[name], k]
158
+ end
159
+ k -= 1
160
+ end
161
+ [nil, 0]
162
+ end
163
+ end
164
+ end