fasthtmlparser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.md +79 -0
- data/README.md +196 -0
- data/fasthtmlparser.gemspec +26 -0
- data/lib/fasthtmlparser/attr.rb +243 -0
- data/lib/fasthtmlparser/attrs.rb +278 -0
- data/lib/fasthtmlparser/chars.rb +258 -0
- data/lib/fasthtmlparser/css.rb +1061 -0
- data/lib/fasthtmlparser/decode.rb +164 -0
- data/lib/fasthtmlparser/entities.rb +2251 -0
- data/lib/fasthtmlparser/html.rb +52 -0
- data/lib/fasthtmlparser/html_document.rb +294 -0
- data/lib/fasthtmlparser/html_node.rb +706 -0
- data/lib/fasthtmlparser/links.rb +499 -0
- data/lib/fasthtmlparser/parser.rb +893 -0
- data/lib/fasthtmlparser/query.rb +221 -0
- data/lib/fasthtmlparser/tags.rb +761 -0
- data/lib/fasthtmlparser/text.rb +280 -0
- data/lib/fasthtmlparser/version.rb +5 -0
- data/lib/fasthtmlparser/xpath.rb +1151 -0
- data/lib/fasthtmlparser/xpath_compiler.rb +627 -0
- data/lib/fasthtmlparser.rb +35 -0
- metadata +66 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FastHtmlParser
|
|
4
|
+
# @api private Reads the attributes of an opening tag one by one: positions in the source.
|
|
5
|
+
class AttrScanner
|
|
6
|
+
attr_reader :name_start, :name_length, :value_start, :value_length
|
|
7
|
+
|
|
8
|
+
# skip '<' and the name; name chars end at whitespace, '/' or '>'
|
|
9
|
+
NAME_END = %r{[^\t-\r />]*+}n
|
|
10
|
+
# between attributes: whitespace and '/'
|
|
11
|
+
GAP = %r{[\t-\r /]*+}n
|
|
12
|
+
# an attribute name ends at whitespace, '=', '/' or '>'
|
|
13
|
+
ATTR_NAME = %r{[^\t-\r =/>]*+}n
|
|
14
|
+
|
|
15
|
+
def initialize(node)
|
|
16
|
+
@src = node.doc.source_bytes
|
|
17
|
+
@end = Attrs.open_tag_end(node)
|
|
18
|
+
@ss = StringScanner.new(@src)
|
|
19
|
+
i = node.start + 1
|
|
20
|
+
if i < @end
|
|
21
|
+
@ss.pos = i
|
|
22
|
+
@ss.skip(NAME_END)
|
|
23
|
+
i = @ss.pos
|
|
24
|
+
i = @end if i > @end
|
|
25
|
+
end
|
|
26
|
+
@pos = i
|
|
27
|
+
@name_start = 0
|
|
28
|
+
@name_length = 0
|
|
29
|
+
@value_start = -1 # -1: no value
|
|
30
|
+
@value_length = 0
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Moves to the next attribute; false at the end of the tag.
|
|
34
|
+
def next?
|
|
35
|
+
s = @src
|
|
36
|
+
e = @end
|
|
37
|
+
ss = @ss
|
|
38
|
+
loop do
|
|
39
|
+
if @pos < e
|
|
40
|
+
ss.pos = @pos
|
|
41
|
+
ss.skip(GAP)
|
|
42
|
+
@pos = ss.pos < e ? ss.pos : e
|
|
43
|
+
end
|
|
44
|
+
return false if @pos >= e
|
|
45
|
+
|
|
46
|
+
@name_start = @pos
|
|
47
|
+
ss.pos = @pos
|
|
48
|
+
ss.skip(ATTR_NAME)
|
|
49
|
+
@pos = ss.pos < e ? ss.pos : e
|
|
50
|
+
@name_length = @pos - @name_start
|
|
51
|
+
if @name_length == 0
|
|
52
|
+
@pos += 1 # a stray '=': skip it
|
|
53
|
+
next
|
|
54
|
+
end
|
|
55
|
+
@value_start, @value_length, @pos = Attrs.read_value(s, @pos, e)
|
|
56
|
+
return true
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def name
|
|
61
|
+
@src.byteslice(@name_start, @name_length)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def raw_value
|
|
65
|
+
@value_start < 0 ? +"".b : @src.byteslice(@value_start, @value_length)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# @api private Attributes are read from the source of the opening tag when they are asked for.
|
|
70
|
+
module Attrs
|
|
71
|
+
CLASS = "class"
|
|
72
|
+
ID = "id"
|
|
73
|
+
# starts a name: after whitespace, a quote, '/' or '='
|
|
74
|
+
NAME_BEFORE = Array.new(256, false)
|
|
75
|
+
" \t\n\v\f\r\"'/=".each_byte { |c| NAME_BEFORE[c] = true }
|
|
76
|
+
NAME_BEFORE.freeze
|
|
77
|
+
# ends a name: at whitespace, '=', '/' or '>'
|
|
78
|
+
NAME_AFTER = Array.new(256, false)
|
|
79
|
+
" \t\n\v\f\r=/>".each_byte { |c| NAME_AFTER[c] = true }
|
|
80
|
+
NAME_AFTER.freeze
|
|
81
|
+
NAME_SIZES = TagNames::NAMES.map(&:bytesize).freeze
|
|
82
|
+
DQ = '"'
|
|
83
|
+
SQ = "'"
|
|
84
|
+
|
|
85
|
+
module_function
|
|
86
|
+
|
|
87
|
+
# The decoded value of the first attribute with the name (lower case); "" when missing or without a value.
|
|
88
|
+
def attr_value(n, name)
|
|
89
|
+
raw = attr_raw(n, name)
|
|
90
|
+
Decode.needs_decoding?(raw) ? Decode.decode(raw, true) : raw
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Whether one of the space-separated classes equals cls (case-insensitive).
|
|
94
|
+
def has_class?(n, cls)
|
|
95
|
+
return false if cls.empty?
|
|
96
|
+
|
|
97
|
+
value = attr_raw(n, CLASS)
|
|
98
|
+
value = Decode.decode(value, true) if Decode.needs_decoding?(value)
|
|
99
|
+
vl = value.bytesize
|
|
100
|
+
cl = cls.bytesize
|
|
101
|
+
return Chars.equal_fold(value, cls) if vl == cl
|
|
102
|
+
return false if cl > vl
|
|
103
|
+
|
|
104
|
+
lv = value.downcase(:ascii)
|
|
105
|
+
lc = cls.downcase(:ascii)
|
|
106
|
+
from = 0
|
|
107
|
+
while from <= vl - cl
|
|
108
|
+
i = lv.index(lc, from)
|
|
109
|
+
return false if i.nil?
|
|
110
|
+
|
|
111
|
+
after = i + cl
|
|
112
|
+
if (i == 0 || Chars.space_byte?(lv.getbyte(i - 1))) && (after == vl || Chars.space_byte?(lv.getbyte(after)))
|
|
113
|
+
return true
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
from = i + 1
|
|
117
|
+
end
|
|
118
|
+
false
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# The raw value of the first attribute with the name (lower case), as a browser reads the tag: quoted or unquoted
|
|
122
|
+
# (href=/p?a=1), "" for an attribute without a value or a missing one.
|
|
123
|
+
def attr_raw(n, attribute)
|
|
124
|
+
# a comment has no attributes, whatever its text looks like
|
|
125
|
+
return +"".b if attribute.empty? || n.key == Parser::COMMENT
|
|
126
|
+
|
|
127
|
+
# the opening tag: "<name attributes", without the closing '>', in lower case
|
|
128
|
+
tag = n.lower_tag
|
|
129
|
+
length = tag.bytesize
|
|
130
|
+
|
|
131
|
+
# The first place where the name stands as a word of its own. Most tags have none.
|
|
132
|
+
alen = attribute.bytesize
|
|
133
|
+
p = tag.index(attribute, 1)
|
|
134
|
+
while p
|
|
135
|
+
after = p + alen
|
|
136
|
+
break if NAME_BEFORE[tag.getbyte(p - 1)] && (after == length || NAME_AFTER[tag.getbyte(after)])
|
|
137
|
+
|
|
138
|
+
p = p + 1 < length ? tag.index(attribute, p + 1) : nil
|
|
139
|
+
end
|
|
140
|
+
return +"".b if p.nil?
|
|
141
|
+
|
|
142
|
+
src = n.doc.source_bytes
|
|
143
|
+
start = n.start
|
|
144
|
+
# That word is the attribute unless it is inside the value of another one (title="see id=5", x= id).
|
|
145
|
+
# Nearly always that is ruled out at once, and the value is read in place.
|
|
146
|
+
key = n.key
|
|
147
|
+
if (key >= 0 && p == NAME_SIZES[key] + 2) || name_for_sure?(tag, p)
|
|
148
|
+
q = p + alen
|
|
149
|
+
while q < length && ((c = tag.getbyte(q)) == 32 || (c >= 9 && c <= 13))
|
|
150
|
+
q += 1
|
|
151
|
+
end
|
|
152
|
+
return +"".b if q >= length || tag.getbyte(q) != 61 # no value
|
|
153
|
+
|
|
154
|
+
q += 1
|
|
155
|
+
while q < length && ((c = tag.getbyte(q)) == 32 || (c >= 9 && c <= 13))
|
|
156
|
+
q += 1
|
|
157
|
+
end
|
|
158
|
+
return +"".b if q >= length
|
|
159
|
+
|
|
160
|
+
if c == 34 || c == 39
|
|
161
|
+
close = tag.index(c == 34 ? DQ : SQ, q + 1)
|
|
162
|
+
return close.nil? ? src.byteslice(start + q + 1, length - q - 1) : src.byteslice(start + q + 1, close - q - 1)
|
|
163
|
+
end
|
|
164
|
+
v = q
|
|
165
|
+
while v < length && !((c = tag.getbyte(v)) == 32 || (c >= 9 && c <= 13))
|
|
166
|
+
v += 1
|
|
167
|
+
end
|
|
168
|
+
return src.byteslice(start + q, v - q)
|
|
169
|
+
end
|
|
170
|
+
pos = start + p
|
|
171
|
+
|
|
172
|
+
# Otherwise the attributes are read one by one, up to the one that starts at that word.
|
|
173
|
+
s = AttrScanner.new(n)
|
|
174
|
+
while s.next?
|
|
175
|
+
# a word the scanner has passed was inside a value: the next one takes its place
|
|
176
|
+
while pos < s.name_start
|
|
177
|
+
pos = next_word(tag, start, attribute, pos + 1)
|
|
178
|
+
return +"".b if pos < 0
|
|
179
|
+
end
|
|
180
|
+
if pos == s.name_start && s.name_length == alen
|
|
181
|
+
return s.value_start < 0 ? +"".b : src.byteslice(s.value_start, s.value_length)
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
+"".b
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# The next place in the opening tag (tag, lower case, at start in the source) at or after the source position
|
|
188
|
+
# from where the name stands as a word of its own, or -1.
|
|
189
|
+
def next_word(tag, start, attribute, from)
|
|
190
|
+
n = attribute.bytesize
|
|
191
|
+
length = tag.bytesize
|
|
192
|
+
i = from - start
|
|
193
|
+
while i < length
|
|
194
|
+
found = tag.index(attribute, i)
|
|
195
|
+
break if found.nil?
|
|
196
|
+
|
|
197
|
+
after = found + n
|
|
198
|
+
return start + found if NAME_BEFORE[tag.getbyte(found - 1)] && (after == length || NAME_AFTER[tag.getbyte(after)])
|
|
199
|
+
|
|
200
|
+
i = found + 1
|
|
201
|
+
end
|
|
202
|
+
-1
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Whether the word at pos of the opening tag ("<name attributes") is an attribute name, judged by the chars before
|
|
206
|
+
# it alone. False means "not sure".
|
|
207
|
+
def name_for_sure?(tag, pos)
|
|
208
|
+
# After whitespace, and not as the unquoted value in "x= word".
|
|
209
|
+
return false unless Chars.space_byte?(tag.getbyte(pos - 1))
|
|
210
|
+
|
|
211
|
+
e = pos
|
|
212
|
+
e -= 1 while e > 0 && Chars.space_byte?(tag.getbyte(e - 1))
|
|
213
|
+
return false if tag.getbyte(e - 1) == 61
|
|
214
|
+
|
|
215
|
+
# Not inside a quoted value: it is enough that the last quote of each kind before the word does not open a value.
|
|
216
|
+
['"', "'"].each do |quote|
|
|
217
|
+
i = tag.rindex(quote, e - 1)
|
|
218
|
+
next if i.nil?
|
|
219
|
+
|
|
220
|
+
j = i
|
|
221
|
+
j -= 1 while j > 0 && Chars.space_byte?(tag.getbyte(j - 1))
|
|
222
|
+
return false if j > 0 && tag.getbyte(j - 1) == 61
|
|
223
|
+
end
|
|
224
|
+
true
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Compares the decoded value; decodes only when the raw value has '&' or CR.
|
|
228
|
+
def attr_equals?(raw, value)
|
|
229
|
+
Decode.needs_decoding?(raw) ? Decode.decode(raw, true) == value : raw == value
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# Whether the decoded value starts with prefix.
|
|
233
|
+
def attr_starts_with?(raw, prefix)
|
|
234
|
+
(Decode.needs_decoding?(raw) ? Decode.decode(raw, true) : raw).start_with?(prefix)
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# The position of the '>' that ends the opening tag.
|
|
238
|
+
def open_tag_end(n)
|
|
239
|
+
end_ = n.inner_start > 0 && n.inner_end > 0 ? n.inner_start - 1 : n.end
|
|
240
|
+
len = n.doc.source_bytes.bytesize
|
|
241
|
+
end_ < len ? end_ : len
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Reads the value that follows an attribute name at pos (before end_): [value start (-1: no '='), length,
|
|
245
|
+
# the position after the value].
|
|
246
|
+
def read_value(s, pos, end_)
|
|
247
|
+
p = pos
|
|
248
|
+
p += 1 while p < end_ && Chars.space_byte?(s.getbyte(p))
|
|
249
|
+
return [-1, 0, pos] if p >= end_ || s.getbyte(p) != 61
|
|
250
|
+
|
|
251
|
+
p += 1
|
|
252
|
+
p += 1 while p < end_ && Chars.space_byte?(s.getbyte(p))
|
|
253
|
+
q = p < end_ ? s.getbyte(p) : nil
|
|
254
|
+
if q == 34 || q == 39
|
|
255
|
+
start = p + 1
|
|
256
|
+
close = start < end_ ? s.index(q == 34 ? '"' : "'", start) : nil
|
|
257
|
+
return [start, end_ - start > 0 ? end_ - start : 0, end_] if close.nil? || close >= end_
|
|
258
|
+
|
|
259
|
+
return [start, close - start, close + 1]
|
|
260
|
+
end
|
|
261
|
+
start = p
|
|
262
|
+
p += 1 while p < end_ && !Chars.space_byte?(s.getbyte(p))
|
|
263
|
+
[start, p - start, p]
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# Whether the element has the attribute (lower case), with or without a value.
|
|
267
|
+
def has_attribute?(n, name)
|
|
268
|
+
return true unless attr_raw(n, name).empty?
|
|
269
|
+
return false if n.key == Parser::COMMENT
|
|
270
|
+
|
|
271
|
+
s = AttrScanner.new(n)
|
|
272
|
+
while s.next?
|
|
273
|
+
return true if Chars.equal_fold(s.name, name)
|
|
274
|
+
end
|
|
275
|
+
false
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
end
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FastHtmlParser
|
|
4
|
+
# @api private Character helpers shared by the parser, the text readers and the query engines. Inside the library
|
|
5
|
+
# every string is binary (ASCII-8BIT) UTF-8 bytes and positions are byte offsets; strings given by the caller are
|
|
6
|
+
# made binary with Chars.b, and strings given back are UTF-8 (Chars.u).
|
|
7
|
+
module Chars
|
|
8
|
+
BINARY = Encoding::BINARY
|
|
9
|
+
UTF_8 = Encoding::UTF_8
|
|
10
|
+
EMPTY = "".b.freeze
|
|
11
|
+
REPLACEMENT = "\xEF\xBF\xBD".b.freeze # U+FFFD
|
|
12
|
+
|
|
13
|
+
# ASCII whitespace as the parser sees it: space, \t, \n, \v, \f, \r.
|
|
14
|
+
SPACE = " \t\n\v\f\r"
|
|
15
|
+
|
|
16
|
+
# One whitespace char as .NET char.IsWhiteSpace sees it, in UTF-8 bytes: ASCII, U+0085, U+00A0, U+1680,
|
|
17
|
+
# U+2000-U+200A, U+2028, U+2029, U+202F, U+205F, U+3000.
|
|
18
|
+
UNICODE_SPACE = "(?:[\\t-\\r ]|\\xC2[\\x85\\xA0]|\\xE1\\x9A\\x80|\\xE2\\x80[\\x80-\\x8A\\xA8\\xA9\\xAF]|" \
|
|
19
|
+
"\\xE2\\x81\\x9F|\\xE3\\x80\\x80)"
|
|
20
|
+
SPACE_RUNS = Regexp.new("#{UNICODE_SPACE}+".b, Regexp::NOENCODING)
|
|
21
|
+
# the first bytes of every whitespace char
|
|
22
|
+
SPACE_BYTE = /[\t-\r \xC2\xE1-\xE3]/n
|
|
23
|
+
CONTINUATION_BYTES = "\x80-\xBF".b.freeze
|
|
24
|
+
FOUR_BYTE_LEADS = "\xF0-\xF7".b.freeze
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
# A string given by the caller, as the library works with it: ASCII-only strings are used as they are (they
|
|
29
|
+
# compare and hash the same in any encoding), others as binary.
|
|
30
|
+
def b(s)
|
|
31
|
+
s = s.to_s unless s.is_a?(String)
|
|
32
|
+
s.ascii_only? ? s : s.b
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# A string given back to the caller: UTF-8. The string must not be one the library keeps (those are frozen).
|
|
36
|
+
def u(s)
|
|
37
|
+
return s if s.encoding == UTF_8
|
|
38
|
+
|
|
39
|
+
s = s.dup if s.frozen?
|
|
40
|
+
s.force_encoding(UTF_8)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def space_byte?(c)
|
|
44
|
+
c == 32 || (c && c >= 9 && c <= 13)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# The byte length of the whitespace char (char.IsWhiteSpace) that starts at i, or 0.
|
|
48
|
+
def space_at(s, i, limit)
|
|
49
|
+
return 0 if i >= limit
|
|
50
|
+
|
|
51
|
+
c = s.getbyte(i)
|
|
52
|
+
return c == 32 || (c >= 9 && c <= 13) ? 1 : 0 if c < 0x80
|
|
53
|
+
|
|
54
|
+
if c == 0xc2
|
|
55
|
+
return 0 if i + 1 >= limit
|
|
56
|
+
|
|
57
|
+
d = s.getbyte(i + 1)
|
|
58
|
+
return d == 0x85 || d == 0xa0 ? 2 : 0
|
|
59
|
+
end
|
|
60
|
+
return 0 if c < 0xe1 || c > 0xe3 || i + 2 >= limit
|
|
61
|
+
|
|
62
|
+
d = s.getbyte(i + 1)
|
|
63
|
+
e = s.getbyte(i + 2)
|
|
64
|
+
return d == 0x9a && e == 0x80 ? 3 : 0 if c == 0xe1
|
|
65
|
+
return d == 0x80 && e == 0x80 ? 3 : 0 if c == 0xe3
|
|
66
|
+
return (e >= 0x80 && e <= 0x8a) || e == 0xa8 || e == 0xa9 || e == 0xaf ? 3 : 0 if d == 0x80
|
|
67
|
+
|
|
68
|
+
d == 0x81 && e == 0x9f ? 3 : 0
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The byte length of the whitespace char (char.IsWhiteSpace) that ends right before end_ (not before from), or 0.
|
|
72
|
+
def space_before(s, end_, from)
|
|
73
|
+
return 0 if end_ <= from
|
|
74
|
+
|
|
75
|
+
c = s.getbyte(end_ - 1)
|
|
76
|
+
return c == 32 || (c >= 9 && c <= 13) ? 1 : 0 if c < 0x80
|
|
77
|
+
return 0 if c > 0xbf
|
|
78
|
+
return c == 0x85 || c == 0xa0 ? 2 : 0 if end_ - 2 >= from && s.getbyte(end_ - 2) == 0xc2
|
|
79
|
+
return space_at(s, end_ - 3, end_) == 3 ? 3 : 0 if end_ - 3 >= from
|
|
80
|
+
|
|
81
|
+
0
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# s[from, to) trimmed of Unicode whitespace (char.IsWhiteSpace), as a new string.
|
|
85
|
+
def trim_range(s, from, to)
|
|
86
|
+
len = s.bytesize
|
|
87
|
+
to = len if to > len
|
|
88
|
+
from = 0 if from < 0
|
|
89
|
+
while from < to
|
|
90
|
+
c = s.getbyte(from)
|
|
91
|
+
if c < 0x80
|
|
92
|
+
break unless c == 32 || (c >= 9 && c <= 13)
|
|
93
|
+
|
|
94
|
+
from += 1
|
|
95
|
+
else
|
|
96
|
+
n = space_at(s, from, to)
|
|
97
|
+
break if n == 0
|
|
98
|
+
|
|
99
|
+
from += n
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
while to > from
|
|
103
|
+
c = s.getbyte(to - 1)
|
|
104
|
+
if c < 0x80
|
|
105
|
+
break unless c == 32 || (c >= 9 && c <= 13)
|
|
106
|
+
|
|
107
|
+
to -= 1
|
|
108
|
+
else
|
|
109
|
+
n = space_before(s, to, from)
|
|
110
|
+
break if n == 0
|
|
111
|
+
|
|
112
|
+
to -= n
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
from >= to ? +"".b : s.byteslice(from, to - from)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Trims Unicode whitespace (char.IsWhiteSpace) from both ends.
|
|
119
|
+
def trim_space(s)
|
|
120
|
+
return s if s.empty?
|
|
121
|
+
|
|
122
|
+
first = s.getbyte(0)
|
|
123
|
+
last = s.getbyte(-1)
|
|
124
|
+
if first > 32 && first < 0xc2 && last > 32 && last < 0x80
|
|
125
|
+
s
|
|
126
|
+
else
|
|
127
|
+
trim_range(s, 0, s.bytesize)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Trims ASCII whitespace from the end.
|
|
132
|
+
def trim_right_ascii(s)
|
|
133
|
+
e = s.bytesize
|
|
134
|
+
e -= 1 while e > 0 && space_byte?(s.getbyte(e - 1))
|
|
135
|
+
e == s.bytesize ? s : s.byteslice(0, e)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Lower case: ASCII, and Unicode for valid UTF-8.
|
|
139
|
+
def lower(s)
|
|
140
|
+
return s.downcase if s.ascii_only?
|
|
141
|
+
|
|
142
|
+
t = s.dup.force_encoding(UTF_8)
|
|
143
|
+
t.valid_encoding? ? t.downcase!&.b || t.b : s.downcase
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Upper case: ASCII, and Unicode for valid UTF-8.
|
|
147
|
+
def upper(s)
|
|
148
|
+
return s.upcase if s.ascii_only?
|
|
149
|
+
|
|
150
|
+
t = s.dup.force_encoding(UTF_8)
|
|
151
|
+
t.valid_encoding? ? t.upcase!&.b || t.b : s.upcase
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def ascii_lower(s)
|
|
155
|
+
s.downcase(:ascii)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def ascii?(s)
|
|
159
|
+
s.ascii_only?
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Case-insensitive comparison (ASCII fast path, Unicode lower case otherwise).
|
|
163
|
+
def equal_fold(a, b)
|
|
164
|
+
return true if a == b
|
|
165
|
+
|
|
166
|
+
return a.bytesize == b.bytesize && a.casecmp?(b) if a.ascii_only? && b.ascii_only?
|
|
167
|
+
return false if utf16_length(a) != utf16_length(b)
|
|
168
|
+
|
|
169
|
+
lower(a) == lower(b)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Whether s[at..] starts with the ASCII string sub, ignoring ASCII case.
|
|
173
|
+
def starts_with_fold?(s, at, sub)
|
|
174
|
+
n = sub.bytesize
|
|
175
|
+
return false if at < 0 || at + n > s.bytesize
|
|
176
|
+
|
|
177
|
+
n == 0 || s.byteslice(at, n).casecmp?(sub)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Case-insensitive (ASCII) search for sub in s[from, to); -1 when it is not there.
|
|
181
|
+
def index_fold(s, sub, from, to)
|
|
182
|
+
n = sub.bytesize
|
|
183
|
+
return from if n == 0
|
|
184
|
+
return -1 if from + n > to
|
|
185
|
+
|
|
186
|
+
i = s.downcase.index(sub.downcase, from)
|
|
187
|
+
i.nil? || i + n > to ? -1 : i
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def hex_byte?(c)
|
|
191
|
+
c && ((c >= 48 && c <= 57) || ((c | 0x20) >= 97 && (c | 0x20) <= 102))
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def alpha_byte?(c)
|
|
195
|
+
c && (c | 0x20) >= 97 && (c | 0x20) <= 122
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def digit_byte?(c)
|
|
199
|
+
c && c >= 48 && c <= 57
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# s[from, to) clamped to the string, "" when from >= to.
|
|
203
|
+
def slice(s, from, to)
|
|
204
|
+
len = s.bytesize
|
|
205
|
+
to = len if to > len
|
|
206
|
+
from = 0 if from < 0
|
|
207
|
+
from >= to ? +"".b : s.byteslice(from, to - from)
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# The length in UTF-16 code units, as .NET counts.
|
|
211
|
+
def utf16_length(s)
|
|
212
|
+
n = s.bytesize
|
|
213
|
+
return n if s.ascii_only?
|
|
214
|
+
|
|
215
|
+
n - s.count(CONTINUATION_BYTES) + s.count(FOUR_BYTE_LEADS)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# The UTF-8 encoding of a code point, as binary.
|
|
219
|
+
def utf8(cp)
|
|
220
|
+
return cp.chr if cp < 0x80
|
|
221
|
+
|
|
222
|
+
[cp].pack("U").force_encoding(BINARY)
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# The chars of a string (one UTF-8 sequence each; a byte of invalid UTF-8 is a char of its own), as binary strings.
|
|
226
|
+
def chars(s)
|
|
227
|
+
return s.chars if s.ascii_only?
|
|
228
|
+
|
|
229
|
+
s.dup.force_encoding(UTF_8).each_char.map { |c| c.force_encoding(BINARY) }
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# The code point of one UTF-8 char.
|
|
233
|
+
def code_point(ch)
|
|
234
|
+
c = ch.getbyte(0)
|
|
235
|
+
n = ch.bytesize
|
|
236
|
+
return c if c < 0x80 || n == 1
|
|
237
|
+
return ((c & 0x1f) << 6) | (ch.getbyte(1) & 0x3f) if c < 0xe0
|
|
238
|
+
return ((c & 0x0f) << 12) | ((ch.getbyte(1) & 0x3f) << 6) | (ch.getbyte(2) & 0x3f) if c < 0xf0
|
|
239
|
+
|
|
240
|
+
((c & 0x07) << 18) | ((ch.getbyte(1) & 0x3f) << 12) | ((ch.getbyte(2) & 0x3f) << 6) | (ch.getbyte(3) & 0x3f)
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# The string as valid UTF-8: invalid sequences become U+FFFD.
|
|
244
|
+
def scrub(s)
|
|
245
|
+
return s if s.ascii_only?
|
|
246
|
+
|
|
247
|
+
t = s.dup.force_encoding(UTF_8)
|
|
248
|
+
t.valid_encoding? ? s : t.scrub("\uFFFD").force_encoding(BINARY)
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# A string in double quotes, for error messages.
|
|
252
|
+
def quote(s)
|
|
253
|
+
t = s.dup.force_encoding(UTF_8)
|
|
254
|
+
t = t.scrub("\uFFFD") unless t.valid_encoding?
|
|
255
|
+
"\"#{t.gsub(/["\\]/) { "\\#{::Regexp.last_match(0)}" }}\""
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
end
|