mdom 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +165 -1
- data/README.md +50 -12
- data/USAGE.md +89 -14
- data/lib/mdom/block_detector.rb +293 -60
- data/lib/mdom/builder.rb +151 -27
- data/lib/mdom/error.rb +0 -4
- data/lib/mdom/html_serializer.rb +19 -1
- data/lib/mdom/inline_parser.rb +402 -169
- data/lib/mdom/markdown_serializer.rb +69 -11
- data/lib/mdom/node.rb +9 -1
- data/lib/mdom/parser.rb +30 -2
- data/lib/mdom/tokenizer.rb +64 -21
- data/lib/mdom/translator.rb +148 -148
- data/lib/mdom/version.rb +1 -1
- metadata +5 -5
data/lib/mdom/inline_parser.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "prism"
|
|
4
|
+
require "strscan"
|
|
4
5
|
|
|
5
6
|
require_relative "node/inline_nodes"
|
|
6
7
|
|
|
@@ -15,6 +16,17 @@ module MDOM
|
|
|
15
16
|
# 产出的是**未求值**的节点:`#{}` 只保存表达式原文,公式只保存 TeX 原文。
|
|
16
17
|
# 块级结构(围栏、块级公式、透明 Ruby……)不属于本类。
|
|
17
18
|
#
|
|
19
|
+
# === 扫描方式
|
|
20
|
+
#
|
|
21
|
+
# 扫描由 +StringScanner+(strscan,Ruby 自带)驱动:主循环用一条正则把**整段
|
|
22
|
+
# 普通字面**一次收下,只在遇到标记字符时才逐字符分派;查找类操作(找配对反引号、
|
|
23
|
+
# 找闭定界符、找 `]` ……)同样交给 +StringScanner+ 的正则搜索,而不是 Ruby 层的
|
|
24
|
+
# 逐字符 +while+ 循环。这样 +String#[]+ 与 +String#<<+ 的调用次数从「每字符一次」
|
|
25
|
+
# 降到「每段一次」。
|
|
26
|
+
#
|
|
27
|
+
# 注意 +StringScanner+ 的位置是**字节**偏移,所以本类内部一律按字节定位,需要
|
|
28
|
+
# 单个字符时用 +String#byteslice+ 取出(+char_before+ 负责向前跨过一个多字节字符)。
|
|
29
|
+
#
|
|
18
30
|
# MDOM::InlineParser.parse("**粗** 与 *斜* 与 `code`")
|
|
19
31
|
class InlineParser
|
|
20
32
|
# 可以被 `\` 转义的字符(CommonMark 的 ASCII 标点,外加 CSM 用到的 `$`、`@`)。
|
|
@@ -32,7 +44,9 @@ module MDOM
|
|
|
32
44
|
|[\u2026\u00B7\u30FB]
|
|
33
45
|
/x
|
|
34
46
|
|
|
35
|
-
|
|
47
|
+
# 提及的名字部分:前导 `@` 已由主循环消费,所以这里不再带 `@`(也不用 `\A`
|
|
48
|
+
# 锚定——+StringScanner#scan+ 就是从当前位置匹配)。
|
|
49
|
+
MENTION_SCAN_RE = /[A-Za-z0-9_][\w-]*(?::[A-Za-z0-9_][\w-]*)?/
|
|
36
50
|
AUTOLINK_RE = /\A(?:[A-Za-z][A-Za-z0-9+.-]*:[^\s<>]*|[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+)\z/
|
|
37
51
|
DESTINATION_RE = /\A(\S*?)(?:\s+"([^"]*)")?\z/
|
|
38
52
|
RATIO_RE = %r{\A(\S+?)/(\S+)\z}
|
|
@@ -49,6 +63,47 @@ module MDOM
|
|
|
49
63
|
["*", "*", Node::Emphasis]
|
|
50
64
|
].freeze
|
|
51
65
|
|
|
66
|
+
# -- 扫描用正则 ----------------------------------------------------------
|
|
67
|
+
#
|
|
68
|
+
# 主循环先用 +LITERAL_RUN+ 整段收下普通字面;匹配不上时当前字符必是标记
|
|
69
|
+
# (两者互补),于是交给 +handle+ 逐字符分派。
|
|
70
|
+
|
|
71
|
+
LITERAL_RUN = /[^\\`*~\[!<$#@\n]+/
|
|
72
|
+
ANY_CHAR = /./m
|
|
73
|
+
|
|
74
|
+
# 预筛的最低文本长度:更短的文本即使退化也只是常数级,不值得为它建索引
|
|
75
|
+
# (见 maybe_closes?)。
|
|
76
|
+
PREFILTER_MIN_BYTES = 4096
|
|
77
|
+
|
|
78
|
+
TICKS = /`+/
|
|
79
|
+
STARS = /\*+/
|
|
80
|
+
TILDES = /~~/
|
|
81
|
+
INS_CLOSE = /\+~/
|
|
82
|
+
DOLLAR = /\$/
|
|
83
|
+
WIKI_CLOSE = /\]\]/
|
|
84
|
+
|
|
85
|
+
# 查找类操作的「跳板」:只停在这些字符上,其余整段跳过。
|
|
86
|
+
EMPHASIS_JUMP = /[\\`\[!*]/
|
|
87
|
+
BRACKET_JUMP = /[\\`\[\]]/
|
|
88
|
+
PAREN_JUMP = /[\\()"']/
|
|
89
|
+
CLOSE_BRACKET = /\]/
|
|
90
|
+
CLOSE_ANGLE = />/
|
|
91
|
+
# 「转义对或目标字符」:找目标字符时用它一次跳过普通字符(见 last_unescaped)。
|
|
92
|
+
ESCAPE_OR_SQUARE = /\\|\]/
|
|
93
|
+
ESCAPE_OR_PAREN = /\\|\)/
|
|
94
|
+
|
|
95
|
+
# 分派要比较的 ASCII 码(扫描位置是字节偏移,用整数比较省掉一次字符串分配)。
|
|
96
|
+
BACKSLASH = "\\".ord
|
|
97
|
+
BACKTICK = "`".ord
|
|
98
|
+
STAR = "*".ord
|
|
99
|
+
OPEN_SQUARE = "[".ord
|
|
100
|
+
CLOSE_SQUARE = "]".ord
|
|
101
|
+
OPEN_PAREN = "(".ord
|
|
102
|
+
CLOSE_PAREN = ")".ord
|
|
103
|
+
EXCLAMATION = "!".ord
|
|
104
|
+
DQUOTE = '"'.ord
|
|
105
|
+
SQUOTE = "'".ord
|
|
106
|
+
|
|
52
107
|
# 解析行内文本,返回行内节点数组。
|
|
53
108
|
#
|
|
54
109
|
# * +styles:+ 为 +false+ 时不再识别样式标签(`*` `**` `~~` `~+`)——样式不得
|
|
@@ -65,17 +120,24 @@ module MDOM
|
|
|
65
120
|
end
|
|
66
121
|
|
|
67
122
|
def parse(text)
|
|
68
|
-
@
|
|
69
|
-
@pos = 0
|
|
123
|
+
@scan = StringScanner.new(text.to_s)
|
|
70
124
|
@nodes = []
|
|
71
125
|
buffer = +""
|
|
126
|
+
@has_close_square = nil
|
|
127
|
+
@has_close_paren = nil
|
|
128
|
+
@close_square_ready = false
|
|
129
|
+
@close_paren_ready = false
|
|
130
|
+
@link_closer_ready = false
|
|
131
|
+
|
|
132
|
+
until @scan.eos?
|
|
133
|
+
run = @scan.scan(LITERAL_RUN)
|
|
134
|
+
if run
|
|
135
|
+
buffer << run
|
|
136
|
+
next
|
|
137
|
+
end
|
|
72
138
|
|
|
73
|
-
|
|
74
|
-
char
|
|
75
|
-
next if handle(char, buffer)
|
|
76
|
-
|
|
77
|
-
buffer << char
|
|
78
|
-
@pos += 1
|
|
139
|
+
char = @scan.getch
|
|
140
|
+
buffer << char unless handle(char, buffer)
|
|
79
141
|
end
|
|
80
142
|
|
|
81
143
|
flush(buffer)
|
|
@@ -84,7 +146,10 @@ module MDOM
|
|
|
84
146
|
|
|
85
147
|
private
|
|
86
148
|
|
|
87
|
-
#
|
|
149
|
+
# 按当前模式分派一个字符(该字符**已经被消费**),返回是否已处理。
|
|
150
|
+
#
|
|
151
|
+
# 契约:处理函数返回 +false+ 时,扫描位置必须与进入时相同——调用方会把
|
|
152
|
+
# +char+ 按字面收下。失败路径统一走 +rewind+。
|
|
88
153
|
def handle(char, buffer)
|
|
89
154
|
return literal_char(char, buffer) if @literal
|
|
90
155
|
|
|
@@ -114,40 +179,40 @@ module MDOM
|
|
|
114
179
|
|
|
115
180
|
# 行内文本里的 `\`:行尾是硬换行,否则按 Markdown 转义语义(CSM §3.11)。
|
|
116
181
|
def escape(buffer)
|
|
117
|
-
following = @
|
|
182
|
+
following = @scan.check(ANY_CHAR)
|
|
118
183
|
return false if following.nil?
|
|
119
184
|
|
|
120
185
|
if following == "\n"
|
|
121
|
-
@
|
|
186
|
+
@scan.skip(ANY_CHAR)
|
|
122
187
|
emit(buffer, Node::HardBreak.new)
|
|
123
188
|
return true
|
|
124
189
|
end
|
|
125
190
|
|
|
126
191
|
return false unless ESCAPABLE.include?(following)
|
|
127
192
|
|
|
193
|
+
@scan.skip(ANY_CHAR)
|
|
128
194
|
buffer << following
|
|
129
|
-
@pos += 2
|
|
130
195
|
true
|
|
131
196
|
end
|
|
132
197
|
|
|
133
198
|
# 软换行折叠(CSM §1.1):前一个字符是 CJK 标点则直接相连,否则折叠为一个空格。
|
|
134
199
|
def line_break(buffer)
|
|
135
|
-
|
|
136
|
-
@pos += 1
|
|
137
|
-
buffer << " " unless cjk_punctuation?(previous)
|
|
200
|
+
buffer << " " unless cjk_punctuation?(char_before(@scan.pos, 1))
|
|
138
201
|
true
|
|
139
202
|
end
|
|
140
203
|
|
|
141
204
|
# `` `代码` `` / ``` ``代码`` ```(CSM §1.5)。内容按字面处理。
|
|
142
205
|
def code_span(buffer)
|
|
143
|
-
|
|
144
|
-
|
|
206
|
+
start = @scan.pos
|
|
207
|
+
ticks = 1 + @scan.skip(TICKS).to_i
|
|
208
|
+
return rewind(start) if ticks > MAX_CODE_TICKS
|
|
145
209
|
|
|
146
|
-
|
|
147
|
-
return
|
|
210
|
+
closing = find_run(TICKS, ticks)
|
|
211
|
+
return rewind(start) if closing.nil?
|
|
148
212
|
|
|
149
|
-
|
|
150
|
-
|
|
213
|
+
body_start = start + ticks - 1
|
|
214
|
+
code = @scan.string.byteslice(body_start, closing - body_start)
|
|
215
|
+
@scan.pos = closing + ticks
|
|
151
216
|
emit(buffer, Node::CodeSpan.new(code))
|
|
152
217
|
true
|
|
153
218
|
end
|
|
@@ -158,23 +223,26 @@ module MDOM
|
|
|
158
223
|
# 闭 `*` 的左边都是非空白、且强调不嵌套(内部不再识别 `*`)。任一不满足就把
|
|
159
224
|
# 整个定界符串按字面收下,不退化成更短的定界符。
|
|
160
225
|
def emphasis(buffer)
|
|
161
|
-
count =
|
|
162
|
-
|
|
163
|
-
|
|
226
|
+
count = 1 + @scan.skip(STARS).to_i
|
|
227
|
+
# 查找会移动扫描位置,失败路径要回到这里,所以先把内容起点定住。
|
|
228
|
+
body_start = @scan.pos
|
|
229
|
+
return literal_emphasis(buffer, count, body_start) unless @styles && count <= 2
|
|
230
|
+
return literal_emphasis(buffer, count, body_start) if blank?(@scan.check(ANY_CHAR))
|
|
164
231
|
|
|
165
|
-
closing = find_emphasis_close(count)
|
|
166
|
-
return literal_emphasis(buffer, count) if closing.nil?
|
|
232
|
+
closing = find_emphasis_close(count, body_start)
|
|
233
|
+
return literal_emphasis(buffer, count, body_start) if closing.nil?
|
|
167
234
|
|
|
168
|
-
inner = @
|
|
169
|
-
@pos = closing + count
|
|
235
|
+
inner = @scan.string.byteslice(body_start, closing - body_start)
|
|
236
|
+
@scan.pos = closing + count
|
|
170
237
|
klass = count == 2 ? Node::Strong : Node::Emphasis
|
|
171
238
|
emit(buffer, klass.new(children: InlineParser.parse(inner, styles: false)))
|
|
172
239
|
true
|
|
173
240
|
end
|
|
174
241
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
@pos
|
|
242
|
+
# 定界符不成立,整串按字面收下(CSM §1.7 不作退化),扫描位置回到 +from+。
|
|
243
|
+
def literal_emphasis(buffer, count, from)
|
|
244
|
+
@scan.pos = from
|
|
245
|
+
buffer << ("*" * count)
|
|
178
246
|
true
|
|
179
247
|
end
|
|
180
248
|
|
|
@@ -182,20 +250,25 @@ module MDOM
|
|
|
182
250
|
def tilde_mark(buffer)
|
|
183
251
|
return false unless @styles
|
|
184
252
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
253
|
+
start = @scan.pos
|
|
254
|
+
if @scan.check(/~/)
|
|
255
|
+
@scan.skip(/~/)
|
|
256
|
+
body_start = @scan.pos
|
|
257
|
+
closing = find_sequence(TILDES, body_start)
|
|
258
|
+
return rewind(start) if closing.nil?
|
|
188
259
|
|
|
189
|
-
inner = @
|
|
190
|
-
@pos = closing + 2
|
|
260
|
+
inner = @scan.string.byteslice(body_start, closing - body_start)
|
|
261
|
+
@scan.pos = closing + 2
|
|
191
262
|
emit(buffer, Node::Del.new(children: InlineParser.parse(inner)))
|
|
192
263
|
true
|
|
193
|
-
elsif @
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
264
|
+
elsif @scan.check(/\+/)
|
|
265
|
+
@scan.skip(/\+/)
|
|
266
|
+
body_start = @scan.pos
|
|
267
|
+
closing = find_sequence(INS_CLOSE, body_start)
|
|
268
|
+
return rewind(start) if closing.nil?
|
|
269
|
+
|
|
270
|
+
inner = @scan.string.byteslice(body_start, closing - body_start)
|
|
271
|
+
@scan.pos = closing + 2
|
|
199
272
|
emit(buffer, Node::Ins.new(children: InlineParser.parse(inner)))
|
|
200
273
|
true
|
|
201
274
|
else
|
|
@@ -203,25 +276,31 @@ module MDOM
|
|
|
203
276
|
end
|
|
204
277
|
end
|
|
205
278
|
|
|
206
|
-
# `[` 家族:WikiLink
|
|
279
|
+
# `[` 家族:WikiLink、进度条、度量、链接/行内引用。整族共用同一个失败回退点——
|
|
280
|
+
# 失败时只消费掉的那个 `[` 由调用方按字面收下。
|
|
207
281
|
def bracket(buffer)
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
elsif @
|
|
213
|
-
|
|
282
|
+
start = @scan.pos
|
|
283
|
+
if @scan.check(/\[/)
|
|
284
|
+
@scan.skip(/\[/)
|
|
285
|
+
wiki_link(buffer, start)
|
|
286
|
+
elsif @scan.check(/->/)
|
|
287
|
+
@scan.skip(/->/)
|
|
288
|
+
bar(buffer, Node::Progress, start)
|
|
289
|
+
elsif @scan.check(/=/)
|
|
290
|
+
@scan.skip(/=/)
|
|
291
|
+
bar(buffer, Node::Meter, start)
|
|
214
292
|
else
|
|
215
293
|
link(buffer)
|
|
216
294
|
end
|
|
217
295
|
end
|
|
218
296
|
|
|
219
|
-
def wiki_link(buffer)
|
|
220
|
-
|
|
221
|
-
|
|
297
|
+
def wiki_link(buffer, start)
|
|
298
|
+
body_start = @scan.pos
|
|
299
|
+
closing = find_sequence(WIKI_CLOSE, body_start)
|
|
300
|
+
return rewind(start) if closing.nil?
|
|
222
301
|
|
|
223
|
-
inner = @
|
|
224
|
-
@pos = closing + 2
|
|
302
|
+
inner = @scan.string.byteslice(body_start, closing - body_start)
|
|
303
|
+
@scan.pos = closing + 2
|
|
225
304
|
target, _, label = inner.partition("|")
|
|
226
305
|
label = nil if label.empty?
|
|
227
306
|
# label 的策略与一般链接的文本相同:最多一个紧邻的样式标签,其余按字面
|
|
@@ -231,79 +310,88 @@ module MDOM
|
|
|
231
310
|
end
|
|
232
311
|
|
|
233
312
|
# `[=m/n]` → `<meter>`,`[->m/n]` → `<progress>`(CSM §1.10)。
|
|
234
|
-
def bar(buffer, klass)
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
return false if closing.nil?
|
|
313
|
+
def bar(buffer, klass, start)
|
|
314
|
+
found = @scan.scan_until(CLOSE_BRACKET)
|
|
315
|
+
return rewind(start) if found.nil?
|
|
238
316
|
|
|
239
|
-
ratio =
|
|
317
|
+
ratio = found.byteslice(0, found.bytesize - 1)
|
|
240
318
|
match = RATIO_RE.match(ratio)
|
|
241
|
-
return
|
|
319
|
+
return rewind(start) if match.nil?
|
|
242
320
|
|
|
243
|
-
@pos = closing + 1
|
|
244
321
|
emit(buffer, klass.new(match[1], match[2], children: literal_nodes(ratio)))
|
|
245
322
|
true
|
|
246
323
|
end
|
|
247
324
|
|
|
248
325
|
# `[文本](uri "title")` → `<a>`;`[文本](@uri)` → `<q>`(CSM §1.9、§2.3)。
|
|
249
326
|
def link(buffer)
|
|
250
|
-
|
|
251
|
-
return
|
|
252
|
-
|
|
327
|
+
start = @scan.pos
|
|
328
|
+
return rewind(start) unless maybe_link?(start - 1)
|
|
329
|
+
|
|
330
|
+
closing = matching_bracket(start - 1)
|
|
331
|
+
return rewind(start) if closing.nil?
|
|
332
|
+
return rewind(start) unless @scan.string.getbyte(closing + 1) == OPEN_PAREN
|
|
253
333
|
|
|
254
334
|
paren = matching_paren(closing + 1)
|
|
255
|
-
return
|
|
335
|
+
return rewind(start) if paren.nil?
|
|
256
336
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
337
|
+
string = @scan.string
|
|
338
|
+
label = string.byteslice(start, closing - start)
|
|
339
|
+
destination = string.byteslice(closing + 2, paren - closing - 2)
|
|
340
|
+
@scan.pos = paren + 1
|
|
260
341
|
emit(buffer, build_link(destination, label))
|
|
261
342
|
true
|
|
262
343
|
end
|
|
263
344
|
|
|
264
345
|
# `` → `<img>`(CSM §2.3)。
|
|
265
346
|
def image(buffer)
|
|
266
|
-
|
|
347
|
+
start = @scan.pos
|
|
348
|
+
return rewind(start) unless @scan.check(/\[/)
|
|
267
349
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
return
|
|
350
|
+
@scan.skip(/\[/)
|
|
351
|
+
open = @scan.pos - 1
|
|
352
|
+
return rewind(start) unless maybe_link?(open)
|
|
353
|
+
|
|
354
|
+
closing = matching_bracket(open)
|
|
355
|
+
return rewind(start) if closing.nil?
|
|
356
|
+
return rewind(start) unless @scan.string.getbyte(closing + 1) == OPEN_PAREN
|
|
271
357
|
|
|
272
358
|
paren = matching_paren(closing + 1)
|
|
273
|
-
return
|
|
359
|
+
return rewind(start) if paren.nil?
|
|
274
360
|
|
|
275
|
-
|
|
276
|
-
|
|
361
|
+
string = @scan.string
|
|
362
|
+
alt = string.byteslice(open + 1, closing - open - 1)
|
|
363
|
+
destination = string.byteslice(closing + 2, paren - closing - 2)
|
|
277
364
|
uri, title = split_destination(destination)
|
|
278
|
-
@pos = paren + 1
|
|
365
|
+
@scan.pos = paren + 1
|
|
279
366
|
emit(buffer, Node::Image.new(uri, alt: alt, title: title, children: literal_nodes(alt)))
|
|
280
367
|
true
|
|
281
368
|
end
|
|
282
369
|
|
|
283
370
|
# `<uri>` / `<mailto:a@b.c>` → `<a>`(CSM §2.3)。
|
|
284
371
|
def autolink(buffer)
|
|
285
|
-
|
|
286
|
-
|
|
372
|
+
start = @scan.pos
|
|
373
|
+
found = @scan.scan_until(CLOSE_ANGLE)
|
|
374
|
+
return rewind(start) if found.nil?
|
|
287
375
|
|
|
288
|
-
inner =
|
|
289
|
-
return
|
|
376
|
+
inner = found.byteslice(0, found.bytesize - 1)
|
|
377
|
+
return rewind(start) if inner.empty? || !AUTOLINK_RE.match?(inner)
|
|
290
378
|
|
|
291
|
-
@pos = closing + 1
|
|
292
379
|
emit(buffer, Node::Link.new(inner, children: literal_nodes(inner)))
|
|
293
380
|
true
|
|
294
381
|
end
|
|
295
382
|
|
|
296
383
|
# `$\TeX$` → 行内 `<span class="math">`(CSM §1.11)。`$$` 属于块级公式,这里按字面处理。
|
|
297
384
|
def math(buffer)
|
|
298
|
-
|
|
385
|
+
start = @scan.pos
|
|
386
|
+
return rewind(start) if @scan.check(DOLLAR)
|
|
299
387
|
|
|
300
|
-
closing = find_sequence(
|
|
301
|
-
return
|
|
388
|
+
closing = find_sequence(DOLLAR, start)
|
|
389
|
+
return rewind(start) if closing.nil?
|
|
302
390
|
|
|
303
|
-
tex = @
|
|
304
|
-
return
|
|
391
|
+
tex = @scan.string.byteslice(start, closing - start)
|
|
392
|
+
return rewind(start) if tex.empty?
|
|
305
393
|
|
|
306
|
-
@pos = closing + 1
|
|
394
|
+
@scan.pos = closing + 1
|
|
307
395
|
emit(buffer, Node::Math.new(tex, children: literal_nodes(tex)))
|
|
308
396
|
true
|
|
309
397
|
end
|
|
@@ -313,29 +401,33 @@ module MDOM
|
|
|
313
401
|
# 边界交给 Prism 判定:把从 `#{` 起的剩余文本包进一个双引号字符串字面量,
|
|
314
402
|
# Prism 给出的插值节点 location 就是 `#{}` 的范围。嵌套 `{}`、字符串与注释
|
|
315
403
|
# 里的 `}` 由 Ruby 自己的词法处理,不需要自研扫描器(CSM §3.12)。
|
|
404
|
+
#
|
|
405
|
+
# 位置算式(`start` 是 `{` 的字节位置,`length` 是 `#{expr}` 的字节长度):
|
|
406
|
+
# `#{expr}` 的起点是 `start - 1`,表达式是它去掉 `#{` 与 `}` 的中间段。
|
|
316
407
|
def interpolation(buffer)
|
|
317
|
-
|
|
408
|
+
start = @scan.pos
|
|
409
|
+
return rewind(start) unless @scan.check(/\{/)
|
|
318
410
|
|
|
319
|
-
|
|
320
|
-
return
|
|
411
|
+
length = interpolation_end(@scan.rest)
|
|
412
|
+
return rewind(start) if length.nil?
|
|
321
413
|
|
|
322
|
-
expr = @
|
|
323
|
-
@pos =
|
|
414
|
+
expr = @scan.string.byteslice(start + 1, length - 3)
|
|
415
|
+
@scan.pos = start + length - 1
|
|
324
416
|
emit(buffer, Node::Interpolation.new(expr))
|
|
325
417
|
true
|
|
326
418
|
end
|
|
327
419
|
|
|
328
420
|
# `@user` / `@project:x` / `@tag:kw`(CSM §2.2)。`:` 前是作用域,其后是名字。
|
|
421
|
+
#
|
|
422
|
+
# 前一个字符是词字符时不算提及(`a@b` 里的 `@` 只是文本)。
|
|
329
423
|
def mention(buffer)
|
|
330
|
-
return false if @
|
|
424
|
+
return false if char_before(@scan.pos, 1)&.match?(/\w/)
|
|
331
425
|
|
|
332
|
-
|
|
333
|
-
return false if
|
|
426
|
+
matched = @scan.scan(MENTION_SCAN_RE)
|
|
427
|
+
return false if matched.nil?
|
|
334
428
|
|
|
335
|
-
|
|
336
|
-
scope
|
|
337
|
-
name = match[2]
|
|
338
|
-
emit(buffer, Node::Mention.new(name || scope, kind: name ? scope : nil))
|
|
429
|
+
scope, _, name = matched.partition(":")
|
|
430
|
+
emit(buffer, Node::Mention.new(name.empty? ? scope : name, kind: name.empty? ? nil : scope))
|
|
339
431
|
true
|
|
340
432
|
end
|
|
341
433
|
|
|
@@ -418,131 +510,268 @@ module MDOM
|
|
|
418
510
|
@nodes << node
|
|
419
511
|
end
|
|
420
512
|
|
|
421
|
-
#
|
|
422
|
-
|
|
423
|
-
|
|
513
|
+
# -- 扫描器辅助 ----------------------------------------------------------
|
|
514
|
+
|
|
515
|
+
# 让扫描位置回到 +position+ 并返回 +false+:处理函数的统一失败出口。
|
|
516
|
+
def rewind(position)
|
|
517
|
+
@scan.pos = position
|
|
518
|
+
false
|
|
519
|
+
end
|
|
520
|
+
|
|
521
|
+
# 把扫描位置向前推进到 +position+,越界时停在文末——`\` 正好是最后一个字符时,
|
|
522
|
+
# 「跳过转义对」的 +2 会多出一格(+StringScanner#pos=+ 对越界值直接抛
|
|
523
|
+
# +RangeError+)。
|
|
524
|
+
def advance_to(position)
|
|
525
|
+
size = @scan.string.bytesize
|
|
526
|
+
@scan.pos = position > size ? size : position
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
# 两级预筛的第一个问题:整段文本里到底有没有 `]`(C 层的 +include?+,一次,极廉)。
|
|
530
|
+
def any_close_square?
|
|
531
|
+
@has_close_square = @scan.string.include?("]") if @has_close_square.nil?
|
|
532
|
+
@has_close_square
|
|
533
|
+
end
|
|
534
|
+
|
|
535
|
+
# 同上,问的是 `)`。
|
|
536
|
+
def any_close_paren?
|
|
537
|
+
@has_close_paren = @scan.string.include?(")") if @has_close_paren.nil?
|
|
538
|
+
@has_close_paren
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
# +open+ 之后是否**可能**存在配对的闭定界符。
|
|
542
|
+
#
|
|
543
|
+
# 两级:先排除「整段根本没有该字符」;只有 +PREFILTER_MIN_BYTES+ 以上的文本才
|
|
544
|
+
# 进一步查「最后一个位置」——收集位置是 Ruby 层循环,短文本(绝大多数行内段)
|
|
545
|
+
# 哪怕退化也只是常数级,不值得付这个固定成本。
|
|
546
|
+
def maybe_closes?(has_any, last_position, open)
|
|
547
|
+
return false unless has_any
|
|
548
|
+
return true if @scan.string.bytesize < PREFILTER_MIN_BYTES
|
|
549
|
+
|
|
550
|
+
last = last_position
|
|
551
|
+
!last.nil? && last > open
|
|
552
|
+
end
|
|
553
|
+
|
|
554
|
+
# 链接/图片的乐观预筛:+open+(`[` 的位置)之后必须存在「未被转义、且后面紧跟
|
|
555
|
+
# `(`」的 `]`——否则既不可能是链接也不可能是图片。
|
|
556
|
+
#
|
|
557
|
+
# 少了这一步,`"[(" * 2000 + ")]" * 2000` 这类输入仍会退化:预筛只看「有没有 `]`」
|
|
558
|
+
# 是拦不住的(`]` 很多),而 `matching_bracket` 仍要一个个扫。
|
|
559
|
+
def maybe_link?(open)
|
|
560
|
+
return false unless any_close_square?
|
|
561
|
+
return true if @scan.string.bytesize < PREFILTER_MIN_BYTES
|
|
562
|
+
|
|
563
|
+
last = last_link_closer
|
|
564
|
+
!last.nil? && last > open
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
# 未被转义的、后面紧跟 `(` 的最后一个 `]` 的位置(没有则 +nil+)。
|
|
568
|
+
def last_link_closer
|
|
569
|
+
unless @link_closer_ready
|
|
570
|
+
@last_link_closer = last_unescaped(CLOSE_SQUARE, ESCAPE_OR_SQUARE) do |position|
|
|
571
|
+
@scan.string.getbyte(position + 1) == OPEN_PAREN
|
|
572
|
+
end
|
|
573
|
+
@link_closer_ready = true
|
|
574
|
+
end
|
|
575
|
+
@last_link_closer
|
|
424
576
|
end
|
|
425
577
|
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
578
|
+
# 未被转义的最后一个 `]` 的字节位置(没有则 +nil+),供 +matching_bracket+ 预筛。
|
|
579
|
+
# 惰性算一次,之后每次预筛只是一次整数比较——不能让它拖慢常见的「配对成功」路径。
|
|
580
|
+
def last_close_square
|
|
581
|
+
unless @close_square_ready
|
|
582
|
+
@last_close_square = last_unescaped(CLOSE_SQUARE, ESCAPE_OR_SQUARE)
|
|
583
|
+
@close_square_ready = true
|
|
584
|
+
end
|
|
585
|
+
@last_close_square
|
|
430
586
|
end
|
|
431
587
|
|
|
432
|
-
#
|
|
433
|
-
def
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
588
|
+
# 未被转义的最后一个 `)` 的位置,供 +matching_paren+ 预筛。
|
|
589
|
+
def last_close_paren
|
|
590
|
+
unless @close_paren_ready
|
|
591
|
+
@last_close_paren = last_unescaped(CLOSE_PAREN, ESCAPE_OR_PAREN)
|
|
592
|
+
@close_paren_ready = true
|
|
593
|
+
end
|
|
594
|
+
@last_close_paren
|
|
595
|
+
end
|
|
439
596
|
|
|
440
|
-
|
|
597
|
+
# 按 +matching_*+ 的扫描规则找「未被转义」的最后一个 +byte+ 的位置。
|
|
598
|
+
#
|
|
599
|
+
# 只求**保守**:代码串、引号内的出现也一并记入(宁可说「有」)——预筛只用于排除
|
|
600
|
+
# 「剩余文本里根本不存在该字符」这种必然无解的情形。
|
|
601
|
+
def last_unescaped(byte, pattern)
|
|
602
|
+
string = @scan.string
|
|
603
|
+
return nil unless string.include?(byte.chr)
|
|
604
|
+
|
|
605
|
+
scanner = StringScanner.new(string)
|
|
606
|
+
last = nil
|
|
607
|
+
while (ahead = scanner.exist?(pattern))
|
|
608
|
+
at = scanner.pos + ahead - scanner.matched_size
|
|
609
|
+
if string.getbyte(at) == byte
|
|
610
|
+
last = at if !block_given? || yield(at)
|
|
611
|
+
scanner.pos = at + 1
|
|
612
|
+
elsif at + 2 <= string.bytesize
|
|
613
|
+
scanner.pos = at + 2 # 转义对:连下一个字符一起跳过
|
|
441
614
|
else
|
|
442
|
-
|
|
615
|
+
scanner.pos = string.bytesize
|
|
443
616
|
end
|
|
444
617
|
end
|
|
618
|
+
last
|
|
619
|
+
end
|
|
620
|
+
|
|
621
|
+
# 从当前位置向后搜索 +pattern+:找到就返回匹配**起点**的字节位置,并把扫描位置
|
|
622
|
+
# 移到匹配之后;找不到返回 +nil+。
|
|
623
|
+
#
|
|
624
|
+
# 用 +exist?+ 而不是 +skip_until+——后者会把「跳过的整段」建成字符串返回,
|
|
625
|
+
# 而这里只要位置。
|
|
626
|
+
def seek(pattern)
|
|
627
|
+
ahead = @scan.exist?(pattern)
|
|
628
|
+
return nil if ahead.nil?
|
|
629
|
+
|
|
630
|
+
start = @scan.pos + ahead - @scan.matched_size
|
|
631
|
+
@scan.pos += ahead
|
|
632
|
+
start
|
|
633
|
+
end
|
|
634
|
+
|
|
635
|
+
# 从当前位置找长度恰好为 +count+ 的 +re+ 串,返回其起点;没有则 +nil+。
|
|
636
|
+
# (不等于 +count+ 的整串会被跳过,与原逐字符实现一致。)
|
|
637
|
+
def find_run(pattern, count)
|
|
638
|
+
cursor = @scan.pos
|
|
639
|
+
while seek(pattern)
|
|
640
|
+
return @scan.pos - @scan.matched_size if @scan.matched_size == count
|
|
641
|
+
end
|
|
642
|
+
@scan.pos = cursor
|
|
643
|
+
nil
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
# 从一个反引号串之后找**等长**的关闭串,返回其之后的位置;找不到 +nil+。
|
|
647
|
+
def skip_code_span_after(ticks)
|
|
648
|
+
while seek(TICKS)
|
|
649
|
+
return @scan.pos if @scan.matched_size == ticks
|
|
650
|
+
end
|
|
445
651
|
nil
|
|
446
652
|
end
|
|
447
653
|
|
|
448
654
|
# 找可以充当闭定界符的 `*`:长度必须与 +count+ 相同,且左侧是非空白
|
|
449
655
|
# (CSM §1.7)。跳过转义、行内代码与链接——它们的 `*` 不算定界符。
|
|
450
|
-
def find_emphasis_close(count)
|
|
451
|
-
|
|
452
|
-
while
|
|
453
|
-
case @
|
|
454
|
-
when
|
|
455
|
-
|
|
456
|
-
when
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
656
|
+
def find_emphasis_close(count, from)
|
|
657
|
+
@scan.pos = from
|
|
658
|
+
while (found = seek(EMPHASIS_JUMP))
|
|
659
|
+
case @scan.string.getbyte(found)
|
|
660
|
+
when BACKSLASH
|
|
661
|
+
advance_to(found + 2)
|
|
662
|
+
when BACKTICK
|
|
663
|
+
@scan.pos = found
|
|
664
|
+
@scan.pos = skip_code_span_after(@scan.skip(TICKS).to_i) || @scan.string.bytesize
|
|
665
|
+
when OPEN_SQUARE, EXCLAMATION
|
|
666
|
+
@scan.pos = link_extent(found) || found + 1
|
|
667
|
+
when STAR
|
|
668
|
+
@scan.pos = found
|
|
669
|
+
run = @scan.skip(STARS).to_i
|
|
670
|
+
return found if run == count && !blank?(char_before(found))
|
|
464
671
|
end
|
|
465
672
|
end
|
|
466
673
|
nil
|
|
467
674
|
end
|
|
468
675
|
|
|
469
|
-
# 找下一个未被转义的 +
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
cursor = found + 1
|
|
676
|
+
# 找下一个未被转义的 +re+ 匹配,返回起点字节位置;找不到 +nil+。
|
|
677
|
+
# 失败时扫描位置落在文末,调用方负责 +rewind+。
|
|
678
|
+
def find_sequence(pattern, from)
|
|
679
|
+
@scan.pos = from
|
|
680
|
+
while (found = seek(pattern))
|
|
681
|
+
return found unless char_before(found) == "\\"
|
|
476
682
|
end
|
|
477
683
|
nil
|
|
478
684
|
end
|
|
479
685
|
|
|
480
686
|
# 从 `[` 的位置找配对的 `]`,跳过嵌套、转义与行内代码。
|
|
481
687
|
def matching_bracket(open)
|
|
688
|
+
# 乐观预筛:`[` 之后没有未被转义的 `]` 时必然找不到配对。少了这一步,
|
|
689
|
+
# 「大量未闭合 `[`」的输入会退化成 O(n²)——每个 `[` 都要扫到文末。
|
|
690
|
+
return nil unless maybe_closes?(any_close_square?, last_close_square, open)
|
|
691
|
+
|
|
482
692
|
depth = 0
|
|
483
|
-
|
|
484
|
-
while
|
|
485
|
-
case @
|
|
486
|
-
when
|
|
487
|
-
|
|
488
|
-
when
|
|
489
|
-
|
|
490
|
-
|
|
693
|
+
@scan.pos = open
|
|
694
|
+
while (found = seek(BRACKET_JUMP))
|
|
695
|
+
case @scan.string.getbyte(found)
|
|
696
|
+
when BACKSLASH
|
|
697
|
+
advance_to(found + 2)
|
|
698
|
+
when BACKTICK
|
|
699
|
+
@scan.pos = found
|
|
700
|
+
@scan.pos = skip_code_span_after(@scan.skip(TICKS).to_i) || @scan.string.bytesize
|
|
701
|
+
when OPEN_SQUARE
|
|
702
|
+
depth += 1
|
|
703
|
+
when CLOSE_SQUARE
|
|
491
704
|
depth -= 1
|
|
492
|
-
return
|
|
493
|
-
|
|
494
|
-
cursor += 1
|
|
495
|
-
else cursor += 1
|
|
705
|
+
return found if depth.zero?
|
|
496
706
|
end
|
|
497
707
|
end
|
|
498
708
|
nil
|
|
499
709
|
end
|
|
500
710
|
|
|
501
711
|
def matching_paren(open)
|
|
502
|
-
|
|
712
|
+
return nil unless maybe_closes?(any_close_paren?, last_close_paren, open)
|
|
713
|
+
|
|
714
|
+
matching_pair(open, OPEN_PAREN, CLOSE_PAREN)
|
|
503
715
|
end
|
|
504
716
|
|
|
505
|
-
# 若 +pos+(`[` 或
|
|
717
|
+
# 若 +pos+(`[` 或 `!` 的位置)处是一个链接/图片,返回它之后的位置;否则 +nil+。
|
|
506
718
|
def link_extent(pos)
|
|
507
|
-
open = @
|
|
508
|
-
return nil unless @
|
|
719
|
+
open = @scan.string.getbyte(pos) == EXCLAMATION ? pos + 1 : pos
|
|
720
|
+
return nil unless @scan.string.getbyte(open) == OPEN_SQUARE
|
|
721
|
+
return nil unless maybe_link?(open)
|
|
509
722
|
|
|
510
723
|
closing = matching_bracket(open)
|
|
511
|
-
return nil if closing.nil? || @
|
|
724
|
+
return nil if closing.nil? || @scan.string.getbyte(closing + 1) != OPEN_PAREN
|
|
512
725
|
|
|
513
726
|
paren = matching_paren(closing + 1)
|
|
514
727
|
paren && paren + 1
|
|
515
728
|
end
|
|
516
729
|
|
|
517
730
|
# 从 +open+(开括号的位置)找配对闭括号;跳过嵌套、反斜杠转义与引号内的括号。
|
|
731
|
+
# 定界符用 ASCII 码比较(扫描位置是字节偏移)。
|
|
518
732
|
def matching_pair(open, opener, closer)
|
|
519
733
|
depth = 0
|
|
520
734
|
quote = nil
|
|
521
|
-
|
|
522
|
-
while
|
|
523
|
-
|
|
524
|
-
if
|
|
525
|
-
|
|
735
|
+
@scan.pos = open
|
|
736
|
+
while (found = seek(PAREN_JUMP))
|
|
737
|
+
byte = @scan.string.getbyte(found)
|
|
738
|
+
if byte == BACKSLASH
|
|
739
|
+
advance_to(found + 2)
|
|
526
740
|
next
|
|
527
741
|
end
|
|
528
742
|
if quote
|
|
529
|
-
quote = nil if
|
|
530
|
-
cursor += 1
|
|
743
|
+
quote = nil if byte == quote
|
|
531
744
|
next
|
|
532
745
|
end
|
|
533
746
|
|
|
534
|
-
case
|
|
535
|
-
when
|
|
747
|
+
case byte
|
|
748
|
+
when DQUOTE, SQUOTE then quote = byte
|
|
536
749
|
when opener then depth += 1
|
|
537
750
|
when closer
|
|
538
751
|
depth -= 1
|
|
539
|
-
return
|
|
752
|
+
return found if depth.zero?
|
|
540
753
|
end
|
|
541
|
-
cursor += 1
|
|
542
754
|
end
|
|
543
755
|
nil
|
|
544
756
|
end
|
|
545
757
|
|
|
758
|
+
# +position+(默认当前位置)之前第 +offset+ 个字符(+0+ = 紧邻的上一个字符,
|
|
759
|
+
# 调用方传 +1+ 表示「跳过紧邻的那一个再看」);越过文本开头时返回 +nil+。
|
|
760
|
+
# UTF-8 下最多回退 3 个字节,不做多余分配。
|
|
761
|
+
def char_before(position = @scan.pos, offset = 0)
|
|
762
|
+
string = @scan.string
|
|
763
|
+
index = position
|
|
764
|
+
(offset + 1).times do
|
|
765
|
+
return nil if index <= 0
|
|
766
|
+
|
|
767
|
+
index -= 1
|
|
768
|
+
index -= 1 while index.positive? && (string.getbyte(index) & 0xC0) == 0x80
|
|
769
|
+
end
|
|
770
|
+
|
|
771
|
+
finish = position
|
|
772
|
+
string.byteslice(index, finish - index)
|
|
773
|
+
end
|
|
774
|
+
|
|
546
775
|
def cjk_punctuation?(char)
|
|
547
776
|
!char.nil? && CJK_PUNCTUATION.match?(char)
|
|
548
777
|
end
|
|
@@ -552,13 +781,17 @@ module MDOM
|
|
|
552
781
|
char.nil? || char.match?(/\s/)
|
|
553
782
|
end
|
|
554
783
|
|
|
555
|
-
# 用 Prism
|
|
784
|
+
# 用 Prism 找出从 +rest+(`#{` 之后的剩余文本)起的 `#{...}` 的**字节长度**;
|
|
785
|
+
# 不是插值则返回 nil。
|
|
556
786
|
#
|
|
557
787
|
# 构造出的字面量里,插值之后若出现裸 `"`,整段就不再是合法 Ruby——Prism
|
|
558
788
|
# 会报错,但错误恢复仍给出插值节点,边界照常可读。代价是每个插值解析一次
|
|
559
789
|
# 剩余文本(各次解析互不影响)。
|
|
560
|
-
def interpolation_end
|
|
561
|
-
|
|
790
|
+
def interpolation_end(rest)
|
|
791
|
+
source = +'"#'
|
|
792
|
+
source << rest
|
|
793
|
+
source << '"'
|
|
794
|
+
result = Prism.parse(source)
|
|
562
795
|
segment = first_embedded_statement(result.value)
|
|
563
796
|
return nil if segment.nil?
|
|
564
797
|
|
|
@@ -566,7 +799,7 @@ module MDOM
|
|
|
566
799
|
# 未闭合的 `#{` 会一直吞到字符串结尾,此时的 slice 不以 `}` 收尾。
|
|
567
800
|
return nil unless slice.end_with?("}")
|
|
568
801
|
|
|
569
|
-
|
|
802
|
+
slice.bytesize
|
|
570
803
|
end
|
|
571
804
|
|
|
572
805
|
# 先序遍历,取第一个字符串插值节点。
|