fasthtmlparser 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,893 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "strscan"
4
+
5
+ module FastHtmlParser
6
+ # @api private A page is scanned once in fast mode: tags are matched to their end tags and nothing else.
7
+ # Well-formed markup needs nothing more. Where the HTML5 optional-end-tag rules matter (<li>one<li>two</ul>), tags
8
+ # are left open inside an element: when its end tag (or the end of the document) finds them on the stack and one of
9
+ # them is a rule target (li, p, td, tr, option, ...), only that part is scanned again in rules mode, from the
10
+ # outermost such tag up to that end tag. Everything before it was parsed right and is kept.
11
+ #
12
+ # Port of ParsingCore.cs from z3l.FastHtmlParser; positions are byte offsets into the UTF-8 source.
13
+ class Parser
14
+ # Nesting deeper than this is flattened, as browsers do (Blink: 512).
15
+ MAX_DEPTH = 512
16
+ # The parts scanned again with the optional-end-tag rules may add up to this many times the source length.
17
+ REPAIR_BUDGET_FACTOR = 8
18
+
19
+ KEYS = TagNames::KEYS
20
+ NAMES = TagNames::NAMES
21
+ COMMENT = KEYS["!--"]
22
+ P = KEYS["p"]
23
+
24
+ # "</script", "</style", ... by tag key, for the elements whose content is raw text (HTML5 RAWTEXT / RCDATA).
25
+ RAW_TEXT_ENDS = Array.new(NAMES.size)
26
+ %w[script style textarea title xmp iframe noembed noframes].each do |n|
27
+ RAW_TEXT_ENDS[KEYS[n]] = "</#{n}".b.freeze
28
+ end
29
+ RAW_TEXT_ENDS.freeze
30
+
31
+ # HTML5 void elements (incl. obsolete ones the parsing rules also treat as void) and declarations.
32
+ VOID_NAMES = %w[area base br col embed hr img input link meta source track wbr basefont bgsound frame keygen param
33
+ isindex !doctype ?xml].freeze
34
+
35
+ # Elements that limit the scope of the rules (HTML5 "has an element in scope").
36
+ SCOPE_STOPS = %w[html body table caption td th template object applet marquee button].freeze
37
+
38
+ # <p> is ended by the start of a block element (HTML5 "close a p element").
39
+ CLOSES_P_NAMES = %w[address article aside blockquote center details dialog dir div dl fieldset figcaption figure
40
+ footer form h1 h2 h3 h4 h5 h6 header hgroup hr listing main menu nav ol p pre search section
41
+ summary table ul li dd dt plaintext xmp].freeze
42
+
43
+ # per tag key: what the parser does with it
44
+ F_VOID = 1
45
+ F_CLOSES_P = 2
46
+ F_RULE = 4
47
+ F_RAW_TEXT = 8
48
+
49
+ # A rule: t (targets) and s (stops) as flags by tag key, k: the target keys.
50
+ Rule = Struct.new(:t, :s, :k)
51
+
52
+ def self.rule(targets, *stops)
53
+ t = Array.new(NAMES.size, false)
54
+ s = Array.new(NAMES.size, false)
55
+ targets.each { |n| t[KEYS[n]] = true }
56
+ stops.each { |list| list.each { |n| s[KEYS[n]] = true } }
57
+ Rule.new(t.freeze, s.freeze, targets.map { |n| KEYS[n] }.freeze).freeze
58
+ end
59
+ private_class_method :rule
60
+
61
+ P_RULE = rule(%w[p], SCOPE_STOPS)
62
+
63
+ RULES = Array.new(NAMES.size)
64
+ RULES[KEYS["li"]] = rule(%w[li], SCOPE_STOPS, %w[ul ol menu])
65
+ RULES[KEYS["dt"]] = RULES[KEYS["dd"]] = rule(%w[dt dd], SCOPE_STOPS, %w[dl])
66
+ table_stops = %w[html body template table]
67
+ RULES[KEYS["td"]] = RULES[KEYS["th"]] = rule(%w[td th], table_stops, %w[tr tbody thead tfoot])
68
+ RULES[KEYS["tr"]] = rule(%w[tr td th], table_stops, %w[tbody thead tfoot])
69
+ section = rule(%w[tbody thead tfoot tr td th caption], table_stops)
70
+ RULES[KEYS["tbody"]] = RULES[KEYS["thead"]] = RULES[KEYS["tfoot"]] = section
71
+ select_stops = %w[html body template select datalist]
72
+ RULES[KEYS["option"]] = rule(%w[option], select_stops, %w[optgroup])
73
+ RULES[KEYS["optgroup"]] = rule(%w[option optgroup], select_stops)
74
+ RULES.freeze
75
+
76
+ FLAGS = Array.new(NAMES.size, 0)
77
+ VOID_NAMES.each { |n| FLAGS[KEYS[n]] |= F_VOID }
78
+ CLOSES_P_NAMES.each { |n| FLAGS[KEYS[n]] |= F_CLOSES_P }
79
+ RAW_TEXT_ENDS.each_with_index { |e, k| FLAGS[k] |= F_RAW_TEXT if e }
80
+ RULES.each_with_index { |r, k| FLAGS[k] |= F_RULE if r }
81
+ FLAGS.freeze
82
+
83
+ IS_RULE_TARGET = Array.new(NAMES.size, false)
84
+ RULES.each { |r| r&.k&.each { |k| IS_RULE_TARGET[k] = true } }
85
+ IS_RULE_TARGET.freeze
86
+
87
+ # Inside a tag only '<', '=' and '>' matter; a quoted value after '=' may contain '<' and '>' (data-x="<p>").
88
+ TAG_BODY = /(?>[^<>=]++|=[\t-\r ]*+(?>"[^"]*+"|'[^']*+')?)*+/n
89
+ # A tag name ends at whitespace or '/' (<br/> is "br", as <br /> is). On the way to the end of the tag it is read
90
+ # up to '<', '=' or '>' too (TAG_BODY reads what follows); no match when it runs into '=' (read byte by byte then).
91
+ NAME_PART = %r{[^\t-\r /<=>]*+(?!=)}n
92
+ LT = "<"
93
+ RAW_TEXT_END_PATTERNS = {}
94
+ RAW_TEXT_ENDS.each { |e| RAW_TEXT_END_PATTERNS[e] = Regexp.new(Regexp.escape(e), Regexp::IGNORECASE | Regexp::NOENCODING) if e }
95
+ RAW_TEXT_END_PATTERNS.freeze
96
+
97
+ # Finds the tag name s[from, to) case-insensitively (ASCII); -1 for a name that is not a known tag.
98
+ def self.lookup_tag_at(s, from, to)
99
+ length = to - from
100
+ return -1 if length <= 0 || length > 32
101
+
102
+ KEYS[s.byteslice(from, length).downcase(:ascii)] || -1
103
+ end
104
+
105
+ # Finds a tag name case-insensitively (ASCII); -1 for a name that is not a known tag.
106
+ def self.lookup_tag(name)
107
+ return -1 if name.empty? || name.bytesize > 32
108
+
109
+ KEYS[name] || KEYS[name.downcase(:ascii)] || -1
110
+ end
111
+
112
+ # Whether the content of the tag is raw text (script, style, textarea, title, ...).
113
+ def self.raw_text?(key)
114
+ key >= 0 && !RAW_TEXT_ENDS[key].nil?
115
+ end
116
+
117
+ # As in HTML5, '<' starts markup only before an ASCII letter, '/', '!' or '?'.
118
+ def self.markup_start?(s, i, limit)
119
+ return false if i + 1 >= limit
120
+
121
+ c = s.getbyte(i + 1)
122
+ ((c | 0x20) >= 97 && (c | 0x20) <= 122) || c == 47 || c == 33 || c == 63
123
+ end
124
+
125
+ # The position of the '<' of the end tag (e.g. "</script") in s[from, limit), or limit if there is none.
126
+ def self.find_raw_text_end(s, from, end_tag, limit)
127
+ n = end_tag.bytesize
128
+ re = RAW_TEXT_END_PATTERNS[end_tag] || Regexp.new(Regexp.escape(end_tag), Regexp::IGNORECASE | Regexp::NOENCODING)
129
+ while from < limit
130
+ pos = s.index(re, from)
131
+ break if pos.nil? || pos + n > limit
132
+
133
+ after = pos + n
134
+ # "</script>", "</script >", "</script/>" end it; "</scripts" does not.
135
+ return pos if after >= limit
136
+
137
+ c = s.getbyte(after)
138
+ return pos if c == 62 || c == 47 || c == 32 || (c >= 9 && c <= 13)
139
+
140
+ from = after
141
+ end
142
+ limit
143
+ end
144
+
145
+ # The position of the '>' that ends a comment whose content starts at content_start (in s[0, limit)). As in
146
+ # HTML5: "<!-->" and "<!--->" are empty comments, "-->" or "--!>" end a comment, otherwise it runs to the end.
147
+ def self.find_comment_end(s, content_start, limit)
148
+ return content_start if content_start < limit && s.getbyte(content_start) == 62
149
+ if content_start + 1 < limit && s.getbyte(content_start) == 45 && s.getbyte(content_start + 1) == 62
150
+ return content_start + 1
151
+ end
152
+
153
+ from = content_start
154
+ len = s.bytesize
155
+ while from < limit
156
+ dashes = from < len ? s.index("--", from) : nil
157
+ break if dashes.nil? || dashes + 2 > limit
158
+
159
+ pos = dashes + 2
160
+ return pos if pos < limit && s.getbyte(pos) == 62
161
+ return pos + 1 if pos + 1 < limit && s.getbyte(pos) == 33 && s.getbyte(pos + 1) == 62
162
+
163
+ from = dashes + 1
164
+ end
165
+ limit - 1
166
+ end
167
+
168
+ # An XML document (RSS, Atom, sitemap, XHTML) is parsed by XML rules: elements end only with an end tag or "/>".
169
+ def self.xml?(src)
170
+ i = 0
171
+ n = src.bytesize
172
+ loop do
173
+ i += 1 while i < n && Chars.space_byte?(src.getbyte(i))
174
+ if i + 3 <= n && src.getbyte(i) == 0xef && src.getbyte(i + 1) == 0xbb && src.getbyte(i + 2) == 0xbf
175
+ i += 3 # a BOM
176
+ next
177
+ end
178
+ break
179
+ end
180
+ n - i >= 5 && src.getbyte(i) == 60 && src.getbyte(i + 1) == 63 && Chars.starts_with_fold?(src, i, "<?xml")
181
+ end
182
+
183
+ # Parses src (binary) into the flat tag list: the tags in document order as parallel arrays
184
+ # [keys, custom names, parents, depths, starts, ends, inner starts, inner ends].
185
+ def self.parse_tags(src)
186
+ return Array.new(8) { [] } unless src.include?("<")
187
+
188
+ st = new(src)
189
+ st.scan_fast
190
+ st.finish
191
+ end
192
+
193
+ def initialize(src)
194
+ @src = src
195
+ @ss = StringScanner.new(src)
196
+ @is_xml = Parser.xml?(src)
197
+ @budget = REPAIR_BUDGET_FACTOR * src.bytesize # bytes that may still be scanned again by repair
198
+ @key = [] # Tag key, or -1 for a custom name
199
+ @customs = [] # the lower-case name of a tag that is not in Tag, "" for the others
200
+ @parents = [] # the index of the enclosing tag, -1 at the top level
201
+ @starts = [] # the opening '<'
202
+ @ends = [] # the last '>': of the end tag, or of the opening tag
203
+ @inner_starts = [] # right after the opening tag
204
+ @inner_ends = [] # the last byte before the end tag; 0 when the tag is not closed
205
+ @depth = []
206
+ @prev_open = [] # the open tags as a linked list in document order
207
+ @self_closed = [] # <x/>: stays in the open list (a later </x> ends it), but nests nothing
208
+ @stack = [] # indexes of the open tags that contain the following ones
209
+ @open_by_key = Array.new(NAMES.size, 0) # per Tag, how many are on the stack: kept only in rules mode
210
+ @open_in_list = Array.new(NAMES.size, 0) # per Tag, how many are in the open list
211
+ @custom_open = {} # custom names (lower case): how many such tags are in the open list
212
+ @count = 0
213
+ @stack_top = 0
214
+ @last_open = -1
215
+ @open_p = 0 # <p> tags on the stack, kept in both modes
216
+ @tag_started = false # scan results
217
+ @tag_start = 0
218
+ end
219
+
220
+ # The end of the document ends every tag still open (a truncated page keeps its text), right before an unfinished
221
+ # tag if the source stops inside one. Returns the arrays.
222
+ def finish
223
+ region_end = @tag_started ? @tag_start - 1 : @src.bytesize
224
+ unless @is_xml
225
+ s = first_misnested(0, @stack_top)
226
+ repair(s, region_end) if s >= 0
227
+ end
228
+ truncate_open_list(0, region_end - 1)
229
+
230
+ out = [@key, @customs, @parents, @depth, @starts, @ends, @inner_starts, @inner_ends]
231
+ count = @count
232
+ out.each { |a| a.slice!(count..) } if count < @key.size
233
+ inner_starts = @inner_starts
234
+ @inner_ends.each_with_index { |e, k| inner_starts[k] = 0 if e == 0 } # not closed: no body
235
+ out
236
+ end
237
+
238
+ # The first scan of the whole page in fast mode: what scan(false, 0, size) does, with the common cases inline and
239
+ # the state in local variables (a method call and an instance variable cost several plain operations). Whatever
240
+ # the inline code does not handle (comments, custom names, the optional-end-tag rules, a misnested end tag, an
241
+ # unfinished tag) is handed to the general code with the state written back.
242
+ def scan_fast
243
+ src = @src
244
+ ss = @ss
245
+ to = src.bytesize
246
+ keys = @key
247
+ customs = @customs
248
+ parents = @parents
249
+ depths = @depth
250
+ starts = @starts
251
+ ends = @ends
252
+ inner_starts = @inner_starts
253
+ inner_ends = @inner_ends
254
+ prev_open = @prev_open
255
+ self_closed = @self_closed
256
+ stack = @stack
257
+ open_in_list = @open_in_list
258
+ is_xml = @is_xml
259
+ empty = Chars::EMPTY
260
+ count = 0
261
+ stack_top = 0
262
+ last_open = -1
263
+ open_p = 0
264
+ i = 0
265
+ while i < to
266
+ lt = src.index(LT, i)
267
+ break if lt.nil?
268
+
269
+ i = lt
270
+ break if i + 1 >= to
271
+
272
+ c = src.getbyte(i + 1)
273
+ tag_start = i + 1
274
+ ss.pos = tag_start
275
+ if c == 47
276
+ p = tag_start + ss.skip(TAG_BODY)
277
+ if p < to && src.getbyte(p) == 62
278
+ name = src.byteslice(tag_start + 1, p - tag_start - 1)
279
+ key = KEYS[name]
280
+ j = last_open
281
+ if key && j >= 0 && keys[j] == key && stack_top > 0 && stack[stack_top - 1] == j
282
+ # the end tag of the innermost open tag
283
+ inner_ends[j] = tag_start - 2
284
+ ends[j] = p
285
+ stack_top -= 1
286
+ open_p -= 1 if key == P
287
+ open_in_list[key] -= 1
288
+ last_open = prev_open[j]
289
+ i = p + 1
290
+ next
291
+ end
292
+ end
293
+ else
294
+ l = c | 0x20
295
+ unless (l >= 97 && l <= 122) || c == 33 || c == 63
296
+ i += 1
297
+ next
298
+ end
299
+ if c != 33
300
+ name = ss.scan(NAME_PART)
301
+ p = ss.pos
302
+ p += ss.skip(TAG_BODY) if src.getbyte(p) != 62
303
+ last = p < to ? src.getbyte(p - 1) : 0
304
+ if name && p < to && src.getbyte(p) == 62 && last < 0x80 && (key = KEYS[name] || (name.bytesize <= 32 && KEYS[name.downcase]))
305
+ flags = is_xml ? 0 : FLAGS[key]
306
+ if flags & F_RULE == 0 || stack_top == 0 || (top = keys[stack[stack_top - 1]]) < 0 || !RULES[key].t[top]
307
+ if open_p == 0 || flags & F_CLOSES_P == 0
308
+ # a known tag that ends no open tag
309
+ if stack_top > 0
310
+ parent = stack[stack_top - 1]
311
+ depth = depths[parent] + 1
312
+ else
313
+ parent = -1
314
+ depth = 0
315
+ end
316
+ customs[count] = empty
317
+ parents[count] = parent
318
+ starts[count] = i
319
+ ends[count] = p
320
+ inner_starts[count] = p + 1
321
+ inner_ends[count] = 0
322
+ keys[count] = key
323
+ depths[count] = depth
324
+ if flags & F_VOID != 0
325
+ prev_open[count] = -1
326
+ self_closed[count] = false
327
+ count += 1
328
+ i = p + 1
329
+ next
330
+ end
331
+ self_closing = last == 47 || (last <= 32 && self_closing?(tag_start, p))
332
+ self_closed[count] = self_closing
333
+ prev_open[count] = last_open
334
+ last_open = count
335
+ open_in_list[key] += 1
336
+ count += 1
337
+ i = p + 1
338
+ next if self_closing
339
+
340
+ if stack_top < MAX_DEPTH
341
+ stack[stack_top] = count - 1
342
+ stack_top += 1
343
+ open_p += 1 if key == P
344
+ end
345
+ # <script>, <style>, ...: continue right before the matching end tag
346
+ i = Parser.find_raw_text_end(src, p + 1, RAW_TEXT_ENDS[key], to) if flags & F_RAW_TEXT != 0
347
+ next
348
+ end
349
+ end
350
+ end
351
+ end
352
+ end
353
+ # anything else: the general code, with the state written back
354
+ @count = count
355
+ @stack_top = stack_top
356
+ @last_open = last_open
357
+ @open_p = open_p
358
+ stop = scan_one(i, to)
359
+ count = @count
360
+ stack_top = @stack_top
361
+ last_open = @last_open
362
+ open_p = @open_p
363
+ if stop < 0
364
+ @tag_started = true
365
+ @tag_start = i + 1
366
+ return
367
+ end
368
+ i = stop
369
+ end
370
+ @count = count
371
+ @stack_top = stack_top
372
+ @last_open = last_open
373
+ @open_p = open_p
374
+ @tag_started = false
375
+ end
376
+
377
+ # Parses src[from, to) (from is outside a tag), adding tags at count. Tags still open at to stay on the stack.
378
+ # Every String method call costs about as much as twenty plain operations, so the common cases are read with as
379
+ # few of them as possible: the name of a start tag and the rest of the tag by two regexps, the name of an end tag
380
+ # looked up untrimmed (a name with spaces around it is not a tag name and takes the slow way).
381
+ def scan(rules, from, to)
382
+ src = @src
383
+ ss = @ss
384
+ tag_started = false
385
+ tag_start = 0
386
+ i = from
387
+ while i < to
388
+ # Outside a tag only '<' matters: jump to it.
389
+ lt = src.index(LT, i)
390
+ break if lt.nil? || lt >= to
391
+
392
+ i = lt
393
+ break if i + 1 >= to
394
+
395
+ c = src.getbyte(i + 1)
396
+ tag_start = i + 1
397
+ if c == 47
398
+ # an end tag; inside a tag only '<', '=' and '>' matter
399
+ ss.pos = tag_start
400
+ p = tag_start + ss.skip(TAG_BODY)
401
+ p = scan_tag_slowly(tag_start, to) if p >= to
402
+ if p < 0
403
+ tag_started = true
404
+ break
405
+ end
406
+ if src.getbyte(p) == 60
407
+ # An unfinished tag: start over at this '<' (it may also start a comment).
408
+ i = p
409
+ next
410
+ end
411
+ end_tag(rules, tag_start, p)
412
+ i = p + 1
413
+ next
414
+ end
415
+
416
+ # As in HTML5, '<' starts markup only before a letter, '/', '!' or '?': "a < b" is text.
417
+ l = c | 0x20
418
+ unless (l >= 97 && l <= 122) || c == 33 || c == 63
419
+ i += 1
420
+ next
421
+ end
422
+ if c == 33 && i + 4 <= to && src.getbyte(i + 2) == 45 && src.getbyte(i + 3) == 45
423
+ # <!-- ... -->: one "!--" tag; its content is not markup, it does not nest anything.
424
+ i = comment(i, to) + 1
425
+ next
426
+ end
427
+
428
+ ss.pos = tag_start
429
+ # nil when the name runs into '=': then the name is read byte by byte
430
+ name = ss.scan(NAME_PART)
431
+ p = ss.pos
432
+ # a tag without attributes ends right after its name
433
+ p += ss.skip(TAG_BODY) if src.getbyte(p) != 62
434
+ p = scan_tag_slowly(tag_start, to) if p >= to
435
+ if p < 0
436
+ tag_started = true
437
+ break
438
+ end
439
+ if src.getbyte(p) == 60
440
+ i = p
441
+ next
442
+ end
443
+ i = start_tag(rules, tag_start, p, to, c, name) + 1
444
+ end
445
+ @tag_started = tag_started
446
+ @tag_start = tag_start
447
+ end
448
+
449
+ private
450
+
451
+ # One step of scan(false, ...) at the '<' at i: the position to go on from, or -1 for a tag that does not end.
452
+ def scan_one(i, to)
453
+ src = @src
454
+ ss = @ss
455
+ c = src.getbyte(i + 1)
456
+ tag_start = i + 1
457
+ ss.pos = tag_start
458
+ if c == 47
459
+ p = tag_start + ss.skip(TAG_BODY)
460
+ p = scan_tag_slowly(tag_start, to) if p >= to
461
+ return -1 if p < 0
462
+ return p if src.getbyte(p) == 60
463
+
464
+ end_tag(false, tag_start, p)
465
+ return p + 1
466
+ end
467
+ l = c | 0x20
468
+ return i + 1 unless (l >= 97 && l <= 122) || c == 33 || c == 63
469
+ return comment(i, to) + 1 if c == 33 && i + 4 <= to && src.getbyte(i + 2) == 45 && src.getbyte(i + 3) == 45
470
+
471
+ name = ss.scan(NAME_PART)
472
+ p = ss.pos
473
+ p += ss.skip(TAG_BODY) if src.getbyte(p) != 62
474
+ p = scan_tag_slowly(tag_start, to) if p >= to
475
+ return -1 if p < 0
476
+ return p if src.getbyte(p) == 60
477
+
478
+ start_tag(false, tag_start, p, to, c, name) + 1
479
+ end
480
+
481
+ # Whether the opening tag that ends at gt (with whitespace before '>') ends with "/".
482
+ def self_closing?(tag_start, gt)
483
+ src = @src
484
+ e = gt
485
+ e -= 1 while e > tag_start && Chars.space_byte?(src.getbyte(e - 1))
486
+ e > tag_start && src.getbyte(e - 1) == 47
487
+ end
488
+
489
+ # The inside of a tag from i, as the fast scan reads it but within to: the position of the '<' or '>' that ends
490
+ # it, or -1 when the tag does not end before to.
491
+ def scan_tag_slowly(i, to)
492
+ src = @src
493
+ while i < to
494
+ c = src.getbyte(i)
495
+ return i if c == 60 || c == 62
496
+
497
+ if c == 61
498
+ q = i + 1
499
+ q += 1 while q < to && Chars.space_byte?(src.getbyte(q))
500
+ if q < to
501
+ quote = src.getbyte(q)
502
+ if quote == 34 || quote == 39
503
+ close = src.index(quote == 34 ? '"' : "'", q + 1)
504
+ i = close if close && close < to
505
+ end
506
+ end
507
+ end
508
+ i += 1
509
+ end
510
+ -1
511
+ end
512
+
513
+ # Adds the comment that starts at i; returns the position of its '>'.
514
+ def comment(i, to)
515
+ comment_end = Parser.find_comment_end(@src, i + 4, to)
516
+ parent = -1
517
+ depth = 0
518
+ if @stack_top > 0
519
+ parent = @stack[@stack_top - 1]
520
+ depth = @depth[parent] + 1
521
+ end
522
+ k = @count
523
+ @key[k] = COMMENT
524
+ @customs[k] = Chars::EMPTY
525
+ @parents[k] = parent
526
+ @starts[k] = i
527
+ @ends[k] = comment_end
528
+ @inner_starts[k] = comment_end + 1
529
+ @inner_ends[k] = 0
530
+ @depth[k] = depth
531
+ @prev_open[k] = -1
532
+ @self_closed[k] = false
533
+ @count = k + 1
534
+ comment_end
535
+ end
536
+
537
+ # Matches the end tag src[tag_start-1 .. gt] to the innermost open tag with that name.
538
+ def end_tag(rules, tag_start, gt)
539
+ src = @src
540
+ from = tag_start + 1
541
+ to = gt
542
+ # Nearly always the name as it stands is a known tag name.
543
+ name = src.byteslice(from, to - from)
544
+ key = KEYS[name]
545
+ unless key
546
+ if from < to && ((b = src.getbyte(to - 1)) <= 32 || b >= 0x80)
547
+ while (n = Chars.space_before(src, to, from)) > 0
548
+ to -= n
549
+ end
550
+ end
551
+ if from < to && ((b = src.getbyte(from)) <= 32 || b >= 0x80)
552
+ while (n = Chars.space_at(src, from, to)) > 0
553
+ from += n
554
+ end
555
+ end
556
+ return if from == to
557
+
558
+ # Resolve the name once, then compare ints instead of a lookup per candidate.
559
+ name = src.byteslice(from, to - from)
560
+ key = KEYS[name] || (to - from <= 32 && KEYS[name.downcase]) || -1
561
+ end
562
+
563
+ # Well-formed markup: the end tag closes the innermost open tag, which is on top of the stack. What match_end
564
+ # and truncate_open_list do for it, without the loops.
565
+ j = @last_open
566
+ if key >= 0 && j >= 0 && @key[j] == key && @stack_top > 0 && @stack[@stack_top - 1] == j
567
+ @inner_ends[j] = tag_start - 2
568
+ @ends[j] = gt
569
+ @stack_top -= 1
570
+ @open_p -= 1 if key == P
571
+ @open_by_key[key] -= 1 if rules
572
+ @open_in_list[key] -= 1
573
+ @last_open = @prev_open[j]
574
+ return
575
+ end
576
+ # The rules may have ended the matched tag itself (<li><div><li>x</div>: the second <li> ends the first one
577
+ # together with its <div>, as in a browser): the end tag is matched again, to what is open now.
578
+ # Once: the part is not scanned a second time.
579
+ return unless match_end(rules, key, name, tag_start, gt, true)
580
+
581
+ match_end(rules, key, name, tag_start, gt, false)
582
+ end
583
+
584
+ # Ends the innermost open tag with the end tag's name. Returns true when it scanned a part again with the
585
+ # optional-end-tag rules instead, and the end tag is to be matched once more.
586
+ def match_end(rules, key, name, tag_start, gt, can_repair)
587
+ # A name that no open tag has is a stray end tag: it is skipped without scanning the open tags.
588
+ closing = nil
589
+ if key >= 0
590
+ return false if @open_in_list[key] <= 0
591
+ else
592
+ return false if @custom_open.empty?
593
+
594
+ closing = Chars.lower(name)
595
+ return false if (@custom_open[closing] || 0) <= 0
596
+ end
597
+
598
+ j = @last_open
599
+ while j >= 0
600
+ if key >= 0
601
+ if @key[j] != key
602
+ j = @prev_open[j]
603
+ next
604
+ end
605
+ elsif @key[j] >= 0 || @customs[j] != closing
606
+ j = @prev_open[j]
607
+ next
608
+ end
609
+
610
+ inner_end = tag_start - 2
611
+
612
+ # Tags left open inside it: if the optional-end-tag rules apply to one of them,
613
+ # scan its part again with the rules (fast mode only; rules mode has applied them).
614
+ if can_repair && !rules && !@is_xml && @stack_top > 0 && @stack[@stack_top - 1] > j
615
+ above = @stack_top - 1
616
+ above -= 1 while above > 0 && @stack[above - 1] > j
617
+ target = first_misnested(above, @stack_top)
618
+ if target >= 0
619
+ repair(target, tag_start - 1)
620
+ return true
621
+ end
622
+ end
623
+
624
+ @inner_ends[j] = inner_end
625
+ @ends[j] = gt
626
+
627
+ while @stack_top > 0 && @stack[@stack_top - 1] >= j
628
+ @stack_top -= 1
629
+ popped = @key[@stack[@stack_top]]
630
+ @open_p -= 1 if popped == P
631
+ @open_by_key[popped] -= 1 if rules && popped >= 0
632
+ end
633
+
634
+ # It leaves the open list with everything opened after it; the tags opened inside it and
635
+ # never closed end right before this end tag, as in a browser.
636
+ truncate_open_list(j, inner_end)
637
+ return false
638
+ end
639
+ false
640
+ end
641
+
642
+ # Adds the tag src[tag_start-1 .. gt]; returns the position to continue from (gt, or before a raw-text end tag).
643
+ # name: the name as the scan read it, nil when it runs into '='.
644
+ def start_tag(rules, tag_start, gt, to, first, name)
645
+ src = @src
646
+ # The name in the inside of the opening tag (trimmed; "!--" for a comment). It starts at tag_start: the first
647
+ # char (first) is a letter, '!' or '?'.
648
+ last = src.getbyte(gt - 1)
649
+ if last >= 0x80 || first == 33 || name.nil?
650
+ end_ = gt
651
+ if last >= 0x80
652
+ while (n = Chars.space_before(src, end_, tag_start)) > 0
653
+ end_ -= n
654
+ end
655
+ end
656
+ if first == 33 && tag_start + 3 <= end_ && src.getbyte(tag_start + 1) == 45 && src.getbyte(tag_start + 2) == 45
657
+ length = 3
658
+ elsif name.nil?
659
+ # The name ends at whitespace or '/': <br/> is "br", as <br /> is.
660
+ e = tag_start
661
+ e += 1 while e < end_ && !Chars.space_byte?(b = src.getbyte(e)) && b != 47
662
+ length = e - tag_start
663
+ else
664
+ length = name.bytesize
665
+ length = end_ - tag_start if tag_start + length > end_
666
+ end
667
+ name = src.byteslice(tag_start, length)
668
+ else
669
+ length = name.bytesize
670
+ end
671
+ key = -1
672
+ custom_name = Chars::EMPTY
673
+ if length > 0
674
+ key = KEYS[name] || (length <= 32 && KEYS[name.downcase]) || -1
675
+ if key < 0
676
+ # Custom names (svg elements, web components, ...): the lower-case name.
677
+ custom_name = Chars.lower(name).freeze
678
+ @custom_open[custom_name] ||= 0
679
+ end
680
+ end
681
+ flags = key >= 0 && !@is_xml ? FLAGS[key] : 0
682
+
683
+ if flags != 0
684
+ # Optional end tags (HTML5): <li> ends an open <li>, <td> an open <td>, ...
685
+ if flags & F_RULE != 0
686
+ rule = RULES[key]
687
+ if rules
688
+ # a target is open (in well-formed markup it is not)
689
+ close_in_scope(rule, tag_start - 2, true) if rule.k.any? { |t| @open_by_key[t] > 0 }
690
+ elsif @stack_top > 0
691
+ # The common case of a left-out end tag is caught right here: the innermost open tag is a target.
692
+ top = @key[@stack[@stack_top - 1]]
693
+ apply_rule_in_fast_mode(rule, tag_start) if top >= 0 && rule.t[top]
694
+ end
695
+ end
696
+
697
+ # A block element ends an open <p>. Usually no <p> is open and this is one compare.
698
+ if @open_p > 0 && flags & F_CLOSES_P != 0
699
+ if rules
700
+ close_in_scope(P_RULE, tag_start - 2, true)
701
+ else
702
+ apply_rule_in_fast_mode(P_RULE, tag_start)
703
+ end
704
+ end
705
+ end
706
+
707
+ # the innermost open tag is the parent
708
+ parent = -1
709
+ depth = 0
710
+ if @stack_top > 0
711
+ parent = @stack[@stack_top - 1]
712
+ depth = @depth[parent] + 1
713
+ end
714
+
715
+ index = @count
716
+ @customs[index] = custom_name
717
+ @parents[index] = parent
718
+ @starts[index] = tag_start - 1
719
+ @ends[index] = gt
720
+ @inner_starts[index] = gt + 1
721
+ @inner_ends[index] = 0
722
+
723
+ # Void elements (<br>, <img>, <meta>, ...) and declarations never contain anything.
724
+ is_void = flags & F_VOID != 0
725
+ self_closing = is_void
726
+ unless self_closing
727
+ if last == 47
728
+ self_closing = true
729
+ elsif last <= 32
730
+ e = gt
731
+ e -= 1 while e > tag_start && Chars.space_byte?(src.getbyte(e - 1))
732
+ self_closing = e > tag_start && src.getbyte(e - 1) == 47
733
+ end
734
+ end
735
+
736
+ @key[index] = key
737
+ @depth[index] = depth
738
+ @self_closed[index] = self_closing && !is_void
739
+ if is_void
740
+ @prev_open[index] = -1
741
+ else
742
+ @prev_open[index] = @last_open
743
+ @last_open = index
744
+ if key >= 0
745
+ @open_in_list[key] += 1
746
+ elsif !custom_name.empty?
747
+ @custom_open[custom_name] += 1
748
+ end
749
+ end
750
+ @count = index + 1
751
+
752
+ unless self_closing
753
+ # Deeper than MAX_DEPTH the tag is not nested: it stays in the open list for its end tag.
754
+ if @stack_top < MAX_DEPTH
755
+ @stack[@stack_top] = index
756
+ @stack_top += 1
757
+ @open_p += 1 if key == P
758
+ @open_by_key[key] += 1 if rules && key >= 0
759
+ end
760
+
761
+ # <script>, <style>, <textarea>, <title>, ...: the content is text, not markup;
762
+ # continue right before the matching end tag.
763
+ return Parser.find_raw_text_end(src, gt + 1, RAW_TEXT_ENDS[key], to) - 1 if flags & F_RAW_TEXT != 0
764
+ end
765
+ gt
766
+ end
767
+
768
+ # ----- implied end tags -----
769
+
770
+ # Among the tags left open at stack positions [from, to), the outermost one that a rule should have ended, or -1.
771
+ def first_misnested(from, to)
772
+ s = from
773
+ while s < to - 1
774
+ key = @key[@stack[s]]
775
+ if key >= 0 && IS_RULE_TARGET[key]
776
+ inner = s + 1
777
+ while inner < to
778
+ inner_key = @key[@stack[inner]]
779
+ return s if inner_key >= 0 && RULES[inner_key]&.t&.[](key)
780
+
781
+ inner += 1
782
+ end
783
+ end
784
+ s += 1
785
+ end
786
+ -1
787
+ end
788
+
789
+ # Scans again with the optional-end-tag rules from the tag at stack position stack_pos up to region_end:
790
+ # the tags from it on are replaced, the ones before it are kept.
791
+ def repair(stack_pos, region_end)
792
+ first = @stack[stack_pos]
793
+ from = @starts[first]
794
+
795
+ # Past the budget the part is left as it is, so that parsing stays linear on any markup.
796
+ cost = region_end - from
797
+ return if cost > @budget
798
+
799
+ @budget -= cost
800
+
801
+ truncate_open_list(first, -1)
802
+ @stack_top = stack_pos
803
+ @count = first
804
+
805
+ @open_by_key.fill(0)
806
+ @open_p = 0
807
+ s = 0
808
+ while s < stack_pos
809
+ key = @key[@stack[s]]
810
+ if key >= 0
811
+ @open_by_key[key] += 1
812
+ @open_p += 1 if key == P
813
+ end
814
+ s += 1
815
+ end
816
+
817
+ scan(true, from, region_end)
818
+ end
819
+
820
+ # Removes the tags from index first on from the open list (they are its tail).
821
+ # With end_ >= 0 the ones that have no end tag of their own end there.
822
+ def truncate_open_list(first, end_)
823
+ k = @last_open
824
+ while k >= first
825
+ key = @key[k]
826
+ if key >= 0
827
+ @open_in_list[key] -= 1
828
+ else
829
+ custom = @customs[k]
830
+ @custom_open[custom] -= 1 if !custom.empty? && @custom_open.key?(custom)
831
+ end
832
+
833
+ if end_ >= 0 && !@self_closed[k] && @inner_ends[k] == 0
834
+ @inner_ends[k] = end_
835
+ @ends[k] = end_
836
+ end
837
+ k = @prev_open[k]
838
+ end
839
+ @last_open = k
840
+ end
841
+
842
+ # The stack position of the outermost target of the rule below the first stop tag, or -1.
843
+ def scope_start(rule)
844
+ from = -1
845
+ s = @stack_top - 1
846
+ while s >= 0
847
+ k = @key[@stack[s]]
848
+ if k >= 0
849
+ if rule.t[k]
850
+ from = s
851
+ elsif rule.s[k]
852
+ break
853
+ end
854
+ end
855
+ s -= 1
856
+ end
857
+ from
858
+ end
859
+
860
+ # Applies the rule for the tag that starts at tag_start. If a tag it ends was itself misnested,
861
+ # that part is scanned again with the rules first.
862
+ def apply_rule_in_fast_mode(rule, tag_start)
863
+ from = scope_start(rule)
864
+ return if from < 0
865
+
866
+ m = first_misnested(from, @stack_top)
867
+ if m >= 0
868
+ repair(m, tag_start - 1)
869
+ from = scope_start(rule) # the part scanned again changed the stack
870
+ end
871
+ close_from(from, tag_start - 2, false)
872
+ end
873
+
874
+ # Ends the outermost target found below the first stop tag together with everything opened inside it.
875
+ def close_in_scope(rule, end_, by_key)
876
+ close_from(scope_start(rule), end_, by_key)
877
+ end
878
+
879
+ # Ends the tag at stack position from (-1: none) together with everything opened inside it.
880
+ def close_from(from, end_, by_key)
881
+ return if from < 0
882
+
883
+ first = @stack[from]
884
+ while @stack_top > from
885
+ @stack_top -= 1
886
+ key = @key[@stack[@stack_top]]
887
+ @open_p -= 1 if key == P
888
+ @open_by_key[key] -= 1 if by_key && key >= 0
889
+ end
890
+ truncate_open_list(first, end_)
891
+ end
892
+ end
893
+ end