fasthtmlparser 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,280 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ # @api private Visible text: tags, comments and invisible content removed, whitespace collapsed, entities decoded.
5
+ module Text
6
+ SPACE_RUNS = Chars::SPACE_RUNS
7
+ SPACE_BYTE = Chars::SPACE_BYTE
8
+ INVISIBLE = {
9
+ "style" => "</style", "script" => "</script", "noembed" => "</noembed", "noscript" => "</noscript",
10
+ "template" => "</template", "noframes" => "</noframes"
11
+ }.freeze
12
+ LT = "<"
13
+ GT = ">"
14
+ SP = " "
15
+
16
+ module_function
17
+
18
+ # s[i] is '<'. Returns the index of the last char of what is not text there: a comment, a tag, or a tag together
19
+ # with invisible content (script, style, ...) up to its end tag. -1: an unterminated tag.
20
+ def skip_markup(s, i)
21
+ n = s.bytesize
22
+ if i + 4 <= n && s.getbyte(i + 1) == 33 && s.getbyte(i + 2) == 45 && s.getbyte(i + 3) == 45
23
+ return Parser.find_comment_end(s, i + 4, n)
24
+ end
25
+
26
+ tag_end = s.index(GT, i + 1)
27
+ return -1 if tag_end.nil?
28
+
29
+ end_tag = invisible_content_end(s, i + 1, tag_end)
30
+ return Parser.find_raw_text_end(s, tag_end + 1, end_tag, n) - 1 if end_tag
31
+
32
+ tag_end
33
+ end
34
+
35
+ # For the inside of an opening tag s[from, to), the end tag ("</script") when the element's content is never
36
+ # visible text (script, style, noscript, template, noframes, noembed), otherwise nil.
37
+ def invisible_content_end(s, from, to)
38
+ return nil if from >= to
39
+
40
+ first = s.getbyte(from) | 0x20
41
+ return nil if first != 0x73 && first != 0x74 && first != 0x6e # s, t, n
42
+
43
+ n = 1
44
+ while from + n < to && n <= 8
45
+ c = s.getbyte(from + n) | 0x20
46
+ break if c < 0x61 || c > 0x7a
47
+
48
+ n += 1
49
+ end
50
+ if from + n < to
51
+ c = s.getbyte(from + n)
52
+ return nil if c != 47 && !Chars.space_byte?(c)
53
+ end
54
+ end_ = INVISIBLE[s.byteslice(from, n).downcase(:ascii)]
55
+ return nil if end_.nil?
56
+
57
+ e = to
58
+ e -= 1 while e > from && Chars.space_byte?(s.getbyte(e - 1))
59
+ return nil if s.getbyte(e - 1) == 47 # <script src="x" />: no content
60
+
61
+ end_
62
+ end
63
+
64
+ # The text runs of s with tags, comments and invisible content removed, whitespace not yet collapsed.
65
+ def text_runs(s)
66
+ lt = s.index(LT)
67
+ return s if lt.nil?
68
+
69
+ n = s.bytesize
70
+ out = String.new(capacity: n, encoding: Encoding::BINARY)
71
+ from = 0
72
+ while lt
73
+ unless Parser.markup_start?(s, lt, n)
74
+ lt = s.index(LT, lt + 1)
75
+ next
76
+ end
77
+ out << s.byteslice(from, lt - from) if lt > from
78
+ last = skip_markup(s, lt)
79
+ return out if last < 0 # an unterminated tag: the rest is markup
80
+
81
+ from = last + 1
82
+ lt = s.index(LT, from)
83
+ end
84
+ return s if from == 0
85
+
86
+ out << s.byteslice(from, n - from) if from < n
87
+ out
88
+ end
89
+
90
+ # Whitespace runs as one ' ', trimmed.
91
+ def collapse(s)
92
+ return s unless SPACE_BYTE.match?(s)
93
+
94
+ t = s.gsub(SPACE_RUNS, SP)
95
+ return t if t.empty?
96
+
97
+ a = t.getbyte(0) == 32 ? 1 : 0
98
+ z = t.bytesize - (t.getbyte(-1) == 32 ? 1 : 0)
99
+ a == 0 && z == t.bytesize ? t : Chars.slice(t, a, z)
100
+ end
101
+
102
+ # The visible text of an HTML fragment.
103
+ def clear_tags(s)
104
+ return +"".b if s.empty?
105
+
106
+ text = collapse(text_runs(s))
107
+ text.include?(Decode::AMP) ? Decode.decode(text, false) : text
108
+ end
109
+
110
+ # Whether the visible text of the body (value) equals expected; n gives the full text.
111
+ def text?(value, expected, n)
112
+ return false if value.empty? || expected.empty?
113
+
114
+ # whitespace is collapsed even where text keeps it (a tag without children), as the .NET version compares
115
+ text = collapse(text_runs(value))
116
+ amp = text.index(Decode::AMP)
117
+ return text == expected if amp.nil?
118
+
119
+ # Text with entities must be decoded before comparing: the full text, when the part before it matches.
120
+ head = text.byteslice(0, amp)
121
+ head = head.byteslice(0, head.bytesize - 1) while head.end_with?(SP)
122
+ expected.start_with?(head) && n.text_bytes == expected
123
+ end
124
+ end
125
+
126
+ # @api private Text with the visible line structure kept (HtmlNode#text_line_breaks).
127
+ class TextLines
128
+ BREAK_SPACE = 1 # table cell: word separator
129
+ BREAK_LINE = 2 # <br> and block elements (div, li, tr, ...): new line
130
+ BREAK_PARAGRAPH = 3 # p, h1-h6: new paragraph (blank line)
131
+ BREAK_SKIP_CONTENT = 4 # script, style, noscript: contents are not text
132
+ # inline tags (b, span, a, ...) are removed, words are not split
133
+
134
+ KEYS = TagNames::KEYS
135
+ BR = KEYS["br"]
136
+
137
+ KINDS = Array.new(TagNames::NAMES.size, 0)
138
+ %w[p h1 h2 h3 h4 h5 h6].each { |t| KINDS[KEYS[t]] = BREAK_PARAGRAPH }
139
+ %w[br div li ul ol dl dt dd table caption tr thead tbody tfoot section article aside nav header footer main address
140
+ blockquote pre hr figure figcaption form fieldset legend details summary dialog option html head body title
141
+ hgroup search menu center dir listing xmp plaintext multicol frameset marquee].each do |t|
142
+ KINDS[KEYS[t]] = BREAK_LINE
143
+ end
144
+ KINDS[KEYS["td"]] = BREAK_SPACE
145
+ KINDS[KEYS["th"]] = BREAK_SPACE
146
+ %w[script style noscript template noframes noembed].each { |t| KINDS[KEYS[t]] = BREAK_SKIP_CONTENT }
147
+ KINDS.freeze
148
+
149
+ SPACE_RUNS = Chars::SPACE_RUNS
150
+ SPACE_BYTE = Chars::SPACE_BYTE
151
+ PLAIN = /[^&<]*+/n
152
+ TAG_NAME_END = %r{[\t\r\n\f /]}n
153
+
154
+ attr_reader :out
155
+
156
+ def initialize
157
+ @out = String.new(encoding: Encoding::BINARY)
158
+ @pending_breaks = 0 # new lines requested since the last text char
159
+ @pending_space = false
160
+ end
161
+
162
+ # Writes text, collapsing whitespace and turning requested breaks into trimmed lines.
163
+ def chars(s)
164
+ unless SPACE_BYTE.match?(s)
165
+ text(s)
166
+ return
167
+ end
168
+ pos = 0
169
+ n = s.bytesize
170
+ while pos < n
171
+ i = s.index(SPACE_RUNS, pos)
172
+ if i.nil?
173
+ text(s.byteslice(pos, n - pos))
174
+ break
175
+ end
176
+ text(s.byteslice(pos, i - pos)) if i > pos
177
+ @pending_space = true
178
+ pos = i + $~[0].bytesize
179
+ end
180
+ end
181
+
182
+ def text(s)
183
+ unless @out.empty?
184
+ # nothing before the first text char: leading breaks and spaces are trimmed
185
+ if @pending_breaks > 0
186
+ @out << (@pending_breaks >= 2 ? "\n\n" : "\n") # at most one blank line
187
+ elsif @pending_space
188
+ @out << " "
189
+ end
190
+ end
191
+ @pending_breaks = 0
192
+ @pending_space = false
193
+ @out << s
194
+ end
195
+
196
+ def break!(kind, known)
197
+ if kind == BREAK_LINE
198
+ if known == BR
199
+ @pending_breaks += 1 # every <br> counts
200
+ elsif @pending_breaks < 1
201
+ @pending_breaks = 1
202
+ end
203
+ elsif @pending_breaks < 2
204
+ @pending_breaks = 2
205
+ end
206
+ end
207
+
208
+ # The text of an HTML fragment with the visible line structure kept. Lines are joined with "\n".
209
+ def self.clear_tags_line_breaks(value)
210
+ return +"".b if value.empty?
211
+
212
+ value = Decode.normalize_newlines(value) # values are read as a browser reads the page
213
+ w = new
214
+ n = value.bytesize
215
+ ss = StringScanner.new(value)
216
+ i = 0
217
+ while i < n
218
+ c = value.getbyte(i)
219
+
220
+ if c == 38
221
+ # '&': decoded chars go through the same whitespace/text logic (&nbsp; collapses, &lt; is plain text)
222
+ decoded, consumed = Decode.decode_reference(value, i + 1)
223
+ if decoded
224
+ w.chars(decoded)
225
+ i += 1 + consumed
226
+ else
227
+ w.chars(Decode::AMP)
228
+ i += 1
229
+ end
230
+ next
231
+ end
232
+
233
+ if c != 60 || !Parser.markup_start?(value, i, n)
234
+ # plain text up to the next '&' or '<'
235
+ ss.pos = i + 1
236
+ ss.skip(PLAIN)
237
+ j = ss.pos
238
+ w.chars(value.byteslice(i, j - i))
239
+ i = j
240
+ next
241
+ end
242
+
243
+ if i + 4 <= n && value.getbyte(i + 1) == 33 && value.getbyte(i + 2) == 45 && value.getbyte(i + 3) == 45
244
+ e = value.index("-->", i + 4)
245
+ break if e.nil?
246
+
247
+ i = e + 3
248
+ next
249
+ end
250
+
251
+ close_tag = value.index(Text::GT, i)
252
+ break if close_tag.nil? # unterminated tag: the rest is markup
253
+
254
+ tag = value.byteslice(i + 1, close_tag - i - 1)
255
+ i = close_tag + 1
256
+
257
+ closing = tag.getbyte(0) == 47
258
+ name = closing ? tag.byteslice(1, tag.bytesize - 1) : tag
259
+ stop = name.index(TAG_NAME_END)
260
+ name = name.byteslice(0, stop) if stop
261
+ known = Parser.lookup_tag(name)
262
+ next if known < 0 # unknown tag: just removed
263
+
264
+ case KINDS[known]
265
+ when BREAK_LINE, BREAK_PARAGRAPH
266
+ w.break!(KINDS[known], known)
267
+ when BREAK_SPACE
268
+ w.chars(Text::SP)
269
+ when BREAK_SKIP_CONTENT
270
+ if !closing && !Chars.trim_right_ascii(tag).end_with?("/")
271
+ # jump to the closing tag; it is removed on the next iteration
272
+ e = Chars.index_fold(value, "</#{TagNames::NAMES[known]}", i, n)
273
+ i = e < 0 ? n : e
274
+ end
275
+ end
276
+ end
277
+ w.out
278
+ end
279
+ end
280
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ VERSION = "0.1.0"
5
+ end