fasthtmlparser 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,278 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ # @api private Reads the attributes of an opening tag one by one: positions in the source.
5
+ class AttrScanner
6
+ attr_reader :name_start, :name_length, :value_start, :value_length
7
+
8
+ # skip '<' and the name; name chars end at whitespace, '/' or '>'
9
+ NAME_END = %r{[^\t-\r />]*+}n
10
+ # between attributes: whitespace and '/'
11
+ GAP = %r{[\t-\r /]*+}n
12
+ # an attribute name ends at whitespace, '=', '/' or '>'
13
+ ATTR_NAME = %r{[^\t-\r =/>]*+}n
14
+
15
+ def initialize(node)
16
+ @src = node.doc.source_bytes
17
+ @end = Attrs.open_tag_end(node)
18
+ @ss = StringScanner.new(@src)
19
+ i = node.start + 1
20
+ if i < @end
21
+ @ss.pos = i
22
+ @ss.skip(NAME_END)
23
+ i = @ss.pos
24
+ i = @end if i > @end
25
+ end
26
+ @pos = i
27
+ @name_start = 0
28
+ @name_length = 0
29
+ @value_start = -1 # -1: no value
30
+ @value_length = 0
31
+ end
32
+
33
+ # Moves to the next attribute; false at the end of the tag.
34
+ def next?
35
+ s = @src
36
+ e = @end
37
+ ss = @ss
38
+ loop do
39
+ if @pos < e
40
+ ss.pos = @pos
41
+ ss.skip(GAP)
42
+ @pos = ss.pos < e ? ss.pos : e
43
+ end
44
+ return false if @pos >= e
45
+
46
+ @name_start = @pos
47
+ ss.pos = @pos
48
+ ss.skip(ATTR_NAME)
49
+ @pos = ss.pos < e ? ss.pos : e
50
+ @name_length = @pos - @name_start
51
+ if @name_length == 0
52
+ @pos += 1 # a stray '=': skip it
53
+ next
54
+ end
55
+ @value_start, @value_length, @pos = Attrs.read_value(s, @pos, e)
56
+ return true
57
+ end
58
+ end
59
+
60
+ def name
61
+ @src.byteslice(@name_start, @name_length)
62
+ end
63
+
64
+ def raw_value
65
+ @value_start < 0 ? +"".b : @src.byteslice(@value_start, @value_length)
66
+ end
67
+ end
68
+
69
+ # @api private Attributes are read from the source of the opening tag when they are asked for.
70
+ module Attrs
71
+ CLASS = "class"
72
+ ID = "id"
73
+ # starts a name: after whitespace, a quote, '/' or '='
74
+ NAME_BEFORE = Array.new(256, false)
75
+ " \t\n\v\f\r\"'/=".each_byte { |c| NAME_BEFORE[c] = true }
76
+ NAME_BEFORE.freeze
77
+ # ends a name: at whitespace, '=', '/' or '>'
78
+ NAME_AFTER = Array.new(256, false)
79
+ " \t\n\v\f\r=/>".each_byte { |c| NAME_AFTER[c] = true }
80
+ NAME_AFTER.freeze
81
+ NAME_SIZES = TagNames::NAMES.map(&:bytesize).freeze
82
+ DQ = '"'
83
+ SQ = "'"
84
+
85
+ module_function
86
+
87
+ # The decoded value of the first attribute with the name (lower case); "" when missing or without a value.
88
+ def attr_value(n, name)
89
+ raw = attr_raw(n, name)
90
+ Decode.needs_decoding?(raw) ? Decode.decode(raw, true) : raw
91
+ end
92
+
93
+ # Whether one of the space-separated classes equals cls (case-insensitive).
94
+ def has_class?(n, cls)
95
+ return false if cls.empty?
96
+
97
+ value = attr_raw(n, CLASS)
98
+ value = Decode.decode(value, true) if Decode.needs_decoding?(value)
99
+ vl = value.bytesize
100
+ cl = cls.bytesize
101
+ return Chars.equal_fold(value, cls) if vl == cl
102
+ return false if cl > vl
103
+
104
+ lv = value.downcase(:ascii)
105
+ lc = cls.downcase(:ascii)
106
+ from = 0
107
+ while from <= vl - cl
108
+ i = lv.index(lc, from)
109
+ return false if i.nil?
110
+
111
+ after = i + cl
112
+ if (i == 0 || Chars.space_byte?(lv.getbyte(i - 1))) && (after == vl || Chars.space_byte?(lv.getbyte(after)))
113
+ return true
114
+ end
115
+
116
+ from = i + 1
117
+ end
118
+ false
119
+ end
120
+
121
+ # The raw value of the first attribute with the name (lower case), as a browser reads the tag: quoted or unquoted
122
+ # (href=/p?a=1), "" for an attribute without a value or a missing one.
123
+ def attr_raw(n, attribute)
124
+ # a comment has no attributes, whatever its text looks like
125
+ return +"".b if attribute.empty? || n.key == Parser::COMMENT
126
+
127
+ # the opening tag: "<name attributes", without the closing '>', in lower case
128
+ tag = n.lower_tag
129
+ length = tag.bytesize
130
+
131
+ # The first place where the name stands as a word of its own. Most tags have none.
132
+ alen = attribute.bytesize
133
+ p = tag.index(attribute, 1)
134
+ while p
135
+ after = p + alen
136
+ break if NAME_BEFORE[tag.getbyte(p - 1)] && (after == length || NAME_AFTER[tag.getbyte(after)])
137
+
138
+ p = p + 1 < length ? tag.index(attribute, p + 1) : nil
139
+ end
140
+ return +"".b if p.nil?
141
+
142
+ src = n.doc.source_bytes
143
+ start = n.start
144
+ # That word is the attribute unless it is inside the value of another one (title="see id=5", x= id).
145
+ # Nearly always that is ruled out at once, and the value is read in place.
146
+ key = n.key
147
+ if (key >= 0 && p == NAME_SIZES[key] + 2) || name_for_sure?(tag, p)
148
+ q = p + alen
149
+ while q < length && ((c = tag.getbyte(q)) == 32 || (c >= 9 && c <= 13))
150
+ q += 1
151
+ end
152
+ return +"".b if q >= length || tag.getbyte(q) != 61 # no value
153
+
154
+ q += 1
155
+ while q < length && ((c = tag.getbyte(q)) == 32 || (c >= 9 && c <= 13))
156
+ q += 1
157
+ end
158
+ return +"".b if q >= length
159
+
160
+ if c == 34 || c == 39
161
+ close = tag.index(c == 34 ? DQ : SQ, q + 1)
162
+ return close.nil? ? src.byteslice(start + q + 1, length - q - 1) : src.byteslice(start + q + 1, close - q - 1)
163
+ end
164
+ v = q
165
+ while v < length && !((c = tag.getbyte(v)) == 32 || (c >= 9 && c <= 13))
166
+ v += 1
167
+ end
168
+ return src.byteslice(start + q, v - q)
169
+ end
170
+ pos = start + p
171
+
172
+ # Otherwise the attributes are read one by one, up to the one that starts at that word.
173
+ s = AttrScanner.new(n)
174
+ while s.next?
175
+ # a word the scanner has passed was inside a value: the next one takes its place
176
+ while pos < s.name_start
177
+ pos = next_word(tag, start, attribute, pos + 1)
178
+ return +"".b if pos < 0
179
+ end
180
+ if pos == s.name_start && s.name_length == alen
181
+ return s.value_start < 0 ? +"".b : src.byteslice(s.value_start, s.value_length)
182
+ end
183
+ end
184
+ +"".b
185
+ end
186
+
187
+ # The next place in the opening tag (tag, lower case, at start in the source) at or after the source position
188
+ # from where the name stands as a word of its own, or -1.
189
+ def next_word(tag, start, attribute, from)
190
+ n = attribute.bytesize
191
+ length = tag.bytesize
192
+ i = from - start
193
+ while i < length
194
+ found = tag.index(attribute, i)
195
+ break if found.nil?
196
+
197
+ after = found + n
198
+ return start + found if NAME_BEFORE[tag.getbyte(found - 1)] && (after == length || NAME_AFTER[tag.getbyte(after)])
199
+
200
+ i = found + 1
201
+ end
202
+ -1
203
+ end
204
+
205
+ # Whether the word at pos of the opening tag ("<name attributes") is an attribute name, judged by the chars before
206
+ # it alone. False means "not sure".
207
+ def name_for_sure?(tag, pos)
208
+ # After whitespace, and not as the unquoted value in "x= word".
209
+ return false unless Chars.space_byte?(tag.getbyte(pos - 1))
210
+
211
+ e = pos
212
+ e -= 1 while e > 0 && Chars.space_byte?(tag.getbyte(e - 1))
213
+ return false if tag.getbyte(e - 1) == 61
214
+
215
+ # Not inside a quoted value: it is enough that the last quote of each kind before the word does not open a value.
216
+ ['"', "'"].each do |quote|
217
+ i = tag.rindex(quote, e - 1)
218
+ next if i.nil?
219
+
220
+ j = i
221
+ j -= 1 while j > 0 && Chars.space_byte?(tag.getbyte(j - 1))
222
+ return false if j > 0 && tag.getbyte(j - 1) == 61
223
+ end
224
+ true
225
+ end
226
+
227
+ # Compares the decoded value; decodes only when the raw value has '&' or CR.
228
+ def attr_equals?(raw, value)
229
+ Decode.needs_decoding?(raw) ? Decode.decode(raw, true) == value : raw == value
230
+ end
231
+
232
+ # Whether the decoded value starts with prefix.
233
+ def attr_starts_with?(raw, prefix)
234
+ (Decode.needs_decoding?(raw) ? Decode.decode(raw, true) : raw).start_with?(prefix)
235
+ end
236
+
237
+ # The position of the '>' that ends the opening tag.
238
+ def open_tag_end(n)
239
+ end_ = n.inner_start > 0 && n.inner_end > 0 ? n.inner_start - 1 : n.end
240
+ len = n.doc.source_bytes.bytesize
241
+ end_ < len ? end_ : len
242
+ end
243
+
244
+ # Reads the value that follows an attribute name at pos (before end_): [value start (-1: no '='), length,
245
+ # the position after the value].
246
+ def read_value(s, pos, end_)
247
+ p = pos
248
+ p += 1 while p < end_ && Chars.space_byte?(s.getbyte(p))
249
+ return [-1, 0, pos] if p >= end_ || s.getbyte(p) != 61
250
+
251
+ p += 1
252
+ p += 1 while p < end_ && Chars.space_byte?(s.getbyte(p))
253
+ q = p < end_ ? s.getbyte(p) : nil
254
+ if q == 34 || q == 39
255
+ start = p + 1
256
+ close = start < end_ ? s.index(q == 34 ? '"' : "'", start) : nil
257
+ return [start, end_ - start > 0 ? end_ - start : 0, end_] if close.nil? || close >= end_
258
+
259
+ return [start, close - start, close + 1]
260
+ end
261
+ start = p
262
+ p += 1 while p < end_ && !Chars.space_byte?(s.getbyte(p))
263
+ [start, p - start, p]
264
+ end
265
+
266
+ # Whether the element has the attribute (lower case), with or without a value.
267
+ def has_attribute?(n, name)
268
+ return true unless attr_raw(n, name).empty?
269
+ return false if n.key == Parser::COMMENT
270
+
271
+ s = AttrScanner.new(n)
272
+ while s.next?
273
+ return true if Chars.equal_fold(s.name, name)
274
+ end
275
+ false
276
+ end
277
+ end
278
+ end
@@ -0,0 +1,258 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ # @api private Character helpers shared by the parser, the text readers and the query engines. Inside the library
5
+ # every string is binary (ASCII-8BIT) UTF-8 bytes and positions are byte offsets; strings given by the caller are
6
+ # made binary with Chars.b, and strings given back are UTF-8 (Chars.u).
7
+ module Chars
8
+ BINARY = Encoding::BINARY
9
+ UTF_8 = Encoding::UTF_8
10
+ EMPTY = "".b.freeze
11
+ REPLACEMENT = "\xEF\xBF\xBD".b.freeze # U+FFFD
12
+
13
+ # ASCII whitespace as the parser sees it: space, \t, \n, \v, \f, \r.
14
+ SPACE = " \t\n\v\f\r"
15
+
16
+ # One whitespace char as .NET char.IsWhiteSpace sees it, in UTF-8 bytes: ASCII, U+0085, U+00A0, U+1680,
17
+ # U+2000-U+200A, U+2028, U+2029, U+202F, U+205F, U+3000.
18
+ UNICODE_SPACE = "(?:[\\t-\\r ]|\\xC2[\\x85\\xA0]|\\xE1\\x9A\\x80|\\xE2\\x80[\\x80-\\x8A\\xA8\\xA9\\xAF]|" \
19
+ "\\xE2\\x81\\x9F|\\xE3\\x80\\x80)"
20
+ SPACE_RUNS = Regexp.new("#{UNICODE_SPACE}+".b, Regexp::NOENCODING)
21
+ # the first bytes of every whitespace char
22
+ SPACE_BYTE = /[\t-\r \xC2\xE1-\xE3]/n
23
+ CONTINUATION_BYTES = "\x80-\xBF".b.freeze
24
+ FOUR_BYTE_LEADS = "\xF0-\xF7".b.freeze
25
+
26
+ module_function
27
+
28
+ # A string given by the caller, as the library works with it: ASCII-only strings are used as they are (they
29
+ # compare and hash the same in any encoding), others as binary.
30
+ def b(s)
31
+ s = s.to_s unless s.is_a?(String)
32
+ s.ascii_only? ? s : s.b
33
+ end
34
+
35
+ # A string given back to the caller: UTF-8. The string must not be one the library keeps (those are frozen).
36
+ def u(s)
37
+ return s if s.encoding == UTF_8
38
+
39
+ s = s.dup if s.frozen?
40
+ s.force_encoding(UTF_8)
41
+ end
42
+
43
+ def space_byte?(c)
44
+ c == 32 || (c && c >= 9 && c <= 13)
45
+ end
46
+
47
+ # The byte length of the whitespace char (char.IsWhiteSpace) that starts at i, or 0.
48
+ def space_at(s, i, limit)
49
+ return 0 if i >= limit
50
+
51
+ c = s.getbyte(i)
52
+ return c == 32 || (c >= 9 && c <= 13) ? 1 : 0 if c < 0x80
53
+
54
+ if c == 0xc2
55
+ return 0 if i + 1 >= limit
56
+
57
+ d = s.getbyte(i + 1)
58
+ return d == 0x85 || d == 0xa0 ? 2 : 0
59
+ end
60
+ return 0 if c < 0xe1 || c > 0xe3 || i + 2 >= limit
61
+
62
+ d = s.getbyte(i + 1)
63
+ e = s.getbyte(i + 2)
64
+ return d == 0x9a && e == 0x80 ? 3 : 0 if c == 0xe1
65
+ return d == 0x80 && e == 0x80 ? 3 : 0 if c == 0xe3
66
+ return (e >= 0x80 && e <= 0x8a) || e == 0xa8 || e == 0xa9 || e == 0xaf ? 3 : 0 if d == 0x80
67
+
68
+ d == 0x81 && e == 0x9f ? 3 : 0
69
+ end
70
+
71
+ # The byte length of the whitespace char (char.IsWhiteSpace) that ends right before end_ (not before from), or 0.
72
+ def space_before(s, end_, from)
73
+ return 0 if end_ <= from
74
+
75
+ c = s.getbyte(end_ - 1)
76
+ return c == 32 || (c >= 9 && c <= 13) ? 1 : 0 if c < 0x80
77
+ return 0 if c > 0xbf
78
+ return c == 0x85 || c == 0xa0 ? 2 : 0 if end_ - 2 >= from && s.getbyte(end_ - 2) == 0xc2
79
+ return space_at(s, end_ - 3, end_) == 3 ? 3 : 0 if end_ - 3 >= from
80
+
81
+ 0
82
+ end
83
+
84
+ # s[from, to) trimmed of Unicode whitespace (char.IsWhiteSpace), as a new string.
85
+ def trim_range(s, from, to)
86
+ len = s.bytesize
87
+ to = len if to > len
88
+ from = 0 if from < 0
89
+ while from < to
90
+ c = s.getbyte(from)
91
+ if c < 0x80
92
+ break unless c == 32 || (c >= 9 && c <= 13)
93
+
94
+ from += 1
95
+ else
96
+ n = space_at(s, from, to)
97
+ break if n == 0
98
+
99
+ from += n
100
+ end
101
+ end
102
+ while to > from
103
+ c = s.getbyte(to - 1)
104
+ if c < 0x80
105
+ break unless c == 32 || (c >= 9 && c <= 13)
106
+
107
+ to -= 1
108
+ else
109
+ n = space_before(s, to, from)
110
+ break if n == 0
111
+
112
+ to -= n
113
+ end
114
+ end
115
+ from >= to ? +"".b : s.byteslice(from, to - from)
116
+ end
117
+
118
+ # Trims Unicode whitespace (char.IsWhiteSpace) from both ends.
119
+ def trim_space(s)
120
+ return s if s.empty?
121
+
122
+ first = s.getbyte(0)
123
+ last = s.getbyte(-1)
124
+ if first > 32 && first < 0xc2 && last > 32 && last < 0x80
125
+ s
126
+ else
127
+ trim_range(s, 0, s.bytesize)
128
+ end
129
+ end
130
+
131
+ # Trims ASCII whitespace from the end.
132
+ def trim_right_ascii(s)
133
+ e = s.bytesize
134
+ e -= 1 while e > 0 && space_byte?(s.getbyte(e - 1))
135
+ e == s.bytesize ? s : s.byteslice(0, e)
136
+ end
137
+
138
+ # Lower case: ASCII, and Unicode for valid UTF-8.
139
+ def lower(s)
140
+ return s.downcase if s.ascii_only?
141
+
142
+ t = s.dup.force_encoding(UTF_8)
143
+ t.valid_encoding? ? t.downcase!&.b || t.b : s.downcase
144
+ end
145
+
146
+ # Upper case: ASCII, and Unicode for valid UTF-8.
147
+ def upper(s)
148
+ return s.upcase if s.ascii_only?
149
+
150
+ t = s.dup.force_encoding(UTF_8)
151
+ t.valid_encoding? ? t.upcase!&.b || t.b : s.upcase
152
+ end
153
+
154
+ def ascii_lower(s)
155
+ s.downcase(:ascii)
156
+ end
157
+
158
+ def ascii?(s)
159
+ s.ascii_only?
160
+ end
161
+
162
+ # Case-insensitive comparison (ASCII fast path, Unicode lower case otherwise).
163
+ def equal_fold(a, b)
164
+ return true if a == b
165
+
166
+ return a.bytesize == b.bytesize && a.casecmp?(b) if a.ascii_only? && b.ascii_only?
167
+ return false if utf16_length(a) != utf16_length(b)
168
+
169
+ lower(a) == lower(b)
170
+ end
171
+
172
+ # Whether s[at..] starts with the ASCII string sub, ignoring ASCII case.
173
+ def starts_with_fold?(s, at, sub)
174
+ n = sub.bytesize
175
+ return false if at < 0 || at + n > s.bytesize
176
+
177
+ n == 0 || s.byteslice(at, n).casecmp?(sub)
178
+ end
179
+
180
+ # Case-insensitive (ASCII) search for sub in s[from, to); -1 when it is not there.
181
+ def index_fold(s, sub, from, to)
182
+ n = sub.bytesize
183
+ return from if n == 0
184
+ return -1 if from + n > to
185
+
186
+ i = s.downcase.index(sub.downcase, from)
187
+ i.nil? || i + n > to ? -1 : i
188
+ end
189
+
190
+ def hex_byte?(c)
191
+ c && ((c >= 48 && c <= 57) || ((c | 0x20) >= 97 && (c | 0x20) <= 102))
192
+ end
193
+
194
+ def alpha_byte?(c)
195
+ c && (c | 0x20) >= 97 && (c | 0x20) <= 122
196
+ end
197
+
198
+ def digit_byte?(c)
199
+ c && c >= 48 && c <= 57
200
+ end
201
+
202
+ # s[from, to) clamped to the string, "" when from >= to.
203
+ def slice(s, from, to)
204
+ len = s.bytesize
205
+ to = len if to > len
206
+ from = 0 if from < 0
207
+ from >= to ? +"".b : s.byteslice(from, to - from)
208
+ end
209
+
210
+ # The length in UTF-16 code units, as .NET counts.
211
+ def utf16_length(s)
212
+ n = s.bytesize
213
+ return n if s.ascii_only?
214
+
215
+ n - s.count(CONTINUATION_BYTES) + s.count(FOUR_BYTE_LEADS)
216
+ end
217
+
218
+ # The UTF-8 encoding of a code point, as binary.
219
+ def utf8(cp)
220
+ return cp.chr if cp < 0x80
221
+
222
+ [cp].pack("U").force_encoding(BINARY)
223
+ end
224
+
225
+ # The chars of a string (one UTF-8 sequence each; a byte of invalid UTF-8 is a char of its own), as binary strings.
226
+ def chars(s)
227
+ return s.chars if s.ascii_only?
228
+
229
+ s.dup.force_encoding(UTF_8).each_char.map { |c| c.force_encoding(BINARY) }
230
+ end
231
+
232
+ # The code point of one UTF-8 char.
233
+ def code_point(ch)
234
+ c = ch.getbyte(0)
235
+ n = ch.bytesize
236
+ return c if c < 0x80 || n == 1
237
+ return ((c & 0x1f) << 6) | (ch.getbyte(1) & 0x3f) if c < 0xe0
238
+ return ((c & 0x0f) << 12) | ((ch.getbyte(1) & 0x3f) << 6) | (ch.getbyte(2) & 0x3f) if c < 0xf0
239
+
240
+ ((c & 0x07) << 18) | ((ch.getbyte(1) & 0x3f) << 12) | ((ch.getbyte(2) & 0x3f) << 6) | (ch.getbyte(3) & 0x3f)
241
+ end
242
+
243
+ # The string as valid UTF-8: invalid sequences become U+FFFD.
244
+ def scrub(s)
245
+ return s if s.ascii_only?
246
+
247
+ t = s.dup.force_encoding(UTF_8)
248
+ t.valid_encoding? ? s : t.scrub("\uFFFD").force_encoding(BINARY)
249
+ end
250
+
251
+ # A string in double quotes, for error messages.
252
+ def quote(s)
253
+ t = s.dup.force_encoding(UTF_8)
254
+ t = t.scrub("\uFFFD") unless t.valid_encoding?
255
+ "\"#{t.gsub(/["\\]/) { "\\#{::Regexp.last_match(0)}" }}\""
256
+ end
257
+ end
258
+ end