maquina_remend 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rdoc_options +16 -0
- data/CHANGELOG.md +31 -0
- data/LICENSE.txt +21 -0
- data/README.md +169 -0
- data/docs/handlers.md +223 -0
- data/docs/repairs.md +391 -0
- data/docs/streaming.md +162 -0
- data/lib/maquina_remend/context.rb +86 -0
- data/lib/maquina_remend/handlers/app_tags.rb +140 -0
- data/lib/maquina_remend/handlers/base.rb +109 -0
- data/lib/maquina_remend/handlers/comparison_operators.rb +33 -0
- data/lib/maquina_remend/handlers/dangling_escape.rb +45 -0
- data/lib/maquina_remend/handlers/emphasis.rb +117 -0
- data/lib/maquina_remend/handlers/html_tags.rb +34 -0
- data/lib/maquina_remend/handlers/inline_code.rb +22 -0
- data/lib/maquina_remend/handlers/links.rb +79 -0
- data/lib/maquina_remend/handlers/math.rb +43 -0
- data/lib/maquina_remend/handlers/setext_heading.rb +46 -0
- data/lib/maquina_remend/handlers/single_tilde.rb +28 -0
- data/lib/maquina_remend/handlers/strikethrough.rb +25 -0
- data/lib/maquina_remend/pipeline.rb +123 -0
- data/lib/maquina_remend/scanner.rb +187 -0
- data/lib/maquina_remend/version.rb +8 -0
- data/lib/maquina_remend.rb +255 -0
- metadata +80 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module MaquinaRemend
|
|
6
|
+
module Handlers
|
|
7
|
+
# Case 25. "20~25°C" is a numeric range, not the start of a strikethrough.
|
|
8
|
+
# Escaping the lone tilde makes that explicit before any parser has to guess.
|
|
9
|
+
# Doubled tildes are left alone — those are case 06's business.
|
|
10
|
+
#
|
|
11
|
+
# Enabled by `single_tilde:`, on by default.
|
|
12
|
+
#
|
|
13
|
+
# ```ruby
|
|
14
|
+
# MaquinaRemend.call("20~25°C") # => "20\\~25°C"
|
|
15
|
+
# ```
|
|
16
|
+
class SingleTilde < Base
|
|
17
|
+
# A single `~` that is neither doubled nor already escaped.
|
|
18
|
+
LONE_TILDE = /(?<![~\\])~(?!~)/
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
def option_key = :single_tilde
|
|
22
|
+
|
|
23
|
+
def repair(_text, scanner)
|
|
24
|
+
escape_matches(scanner, LONE_TILDE)
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module MaquinaRemend
|
|
6
|
+
module Handlers
|
|
7
|
+
# Case 06. Double tildes only. A lone tilde is case 25's problem, and
|
|
8
|
+
# treating it as strikethrough is exactly the false positive that case
|
|
9
|
+
# exists to prevent.
|
|
10
|
+
#
|
|
11
|
+
# Enabled by `strikethrough:`, on by default.
|
|
12
|
+
#
|
|
13
|
+
# ```ruby
|
|
14
|
+
# MaquinaRemend.call("~~struck") # => "~~struck~~"
|
|
15
|
+
# ```
|
|
16
|
+
class Strikethrough < Base
|
|
17
|
+
private
|
|
18
|
+
def option_key = :strikethrough
|
|
19
|
+
|
|
20
|
+
def repair(text, scanner)
|
|
21
|
+
scanner.masked_paragraph.scan("~~").length.odd? ? text + "~~" : text
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "handlers/app_tags"
|
|
4
|
+
require_relative "handlers/dangling_escape"
|
|
5
|
+
require_relative "handlers/html_tags"
|
|
6
|
+
require_relative "handlers/inline_code"
|
|
7
|
+
require_relative "handlers/emphasis"
|
|
8
|
+
require_relative "handlers/strikethrough"
|
|
9
|
+
require_relative "handlers/links"
|
|
10
|
+
require_relative "handlers/math"
|
|
11
|
+
require_relative "handlers/setext_heading"
|
|
12
|
+
require_relative "handlers/comparison_operators"
|
|
13
|
+
require_relative "handlers/single_tilde"
|
|
14
|
+
|
|
15
|
+
module MaquinaRemend
|
|
16
|
+
# Runs the built-in handlers in a fixed order, then any handlers the caller
|
|
17
|
+
# registered. Nothing runs at all when the cursor is inside a fenced code
|
|
18
|
+
# block: a fence is the one place where broken-looking markup is the content.
|
|
19
|
+
#
|
|
20
|
+
# MaquinaRemend.call builds one of these per call and throws it away; it holds
|
|
21
|
+
# no state between buffers. Constructing it directly is worth it only to
|
|
22
|
+
# validate an option hash once and reuse it across many buffers:
|
|
23
|
+
#
|
|
24
|
+
# ```ruby
|
|
25
|
+
# pipeline = MaquinaRemend::Pipeline.new(inline_math: true)
|
|
26
|
+
# deltas.map { |buffer| pipeline.call(buffer) }
|
|
27
|
+
# ```
|
|
28
|
+
class Pipeline
|
|
29
|
+
# Order is load-bearing, and two of these were learned from replaying real
|
|
30
|
+
# streamed output rather than from the spec:
|
|
31
|
+
#
|
|
32
|
+
# * Links run before Emphasis. Emphasis appending "**" into a half-typed
|
|
33
|
+
# "[label](" puts the closer inside the destination, where the link
|
|
34
|
+
# handler then eats it as a partial URL - and appends it again next pass.
|
|
35
|
+
# * DanglingEscape runs first. A trailing backslash escapes whatever any
|
|
36
|
+
# later handler appends.
|
|
37
|
+
# * AppTags runs last. Its closing tag has to sit outside every other
|
|
38
|
+
# completion, or Emphasis closes a bold run after `</thinking>` instead of
|
|
39
|
+
# inside it.
|
|
40
|
+
BUILT_INS = [
|
|
41
|
+
Handlers::DanglingEscape,
|
|
42
|
+
Handlers::HtmlTags,
|
|
43
|
+
Handlers::InlineCode,
|
|
44
|
+
Handlers::Links,
|
|
45
|
+
Handlers::Math,
|
|
46
|
+
Handlers::Emphasis,
|
|
47
|
+
Handlers::Strikethrough,
|
|
48
|
+
Handlers::SetextHeading,
|
|
49
|
+
Handlers::ComparisonOperators,
|
|
50
|
+
Handlers::SingleTilde,
|
|
51
|
+
Handlers::AppTags
|
|
52
|
+
].freeze
|
|
53
|
+
|
|
54
|
+
# The resolved option hash: MaquinaRemend::DEFAULTS merged with whatever the
|
|
55
|
+
# caller passed. Frozen defaults, so this is the merged copy.
|
|
56
|
+
attr_reader :options
|
|
57
|
+
|
|
58
|
+
# Validates `options` and builds a pipeline.
|
|
59
|
+
#
|
|
60
|
+
# * `options` — any subset of MaquinaRemend::DEFAULTS.
|
|
61
|
+
# * Raises MaquinaRemend::UnknownOption for a key outside
|
|
62
|
+
# MaquinaRemend::DEFAULTS, or a `link_mode:` outside
|
|
63
|
+
# MaquinaRemend::LINK_MODES.
|
|
64
|
+
#
|
|
65
|
+
# Validation happens here rather than at repair time so that a bad option is
|
|
66
|
+
# a startup error for the host, not a surprise halfway through a stream.
|
|
67
|
+
def initialize(**options)
|
|
68
|
+
unknown = options.keys - DEFAULTS.keys
|
|
69
|
+
raise UnknownOption, "unknown option(s): #{unknown.join(", ")}" if unknown.any?
|
|
70
|
+
|
|
71
|
+
@options = DEFAULTS.merge(options)
|
|
72
|
+
validate_link_mode
|
|
73
|
+
validate_app_tags
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Repairs `markdown` and returns the result. This is what
|
|
77
|
+
# MaquinaRemend.call delegates to.
|
|
78
|
+
#
|
|
79
|
+
# `nil` and the empty string are passed straight back. Otherwise each
|
|
80
|
+
# built-in in BUILT_INS gets a turn, and the buffer is rescanned after any
|
|
81
|
+
# handler that changed it — a handler must not read a view of the text that
|
|
82
|
+
# a previous handler has already invalidated by appending to its tail.
|
|
83
|
+
# Custom handlers from the `handlers:` option run last.
|
|
84
|
+
def call(markdown)
|
|
85
|
+
return markdown if markdown.nil? || markdown.empty?
|
|
86
|
+
|
|
87
|
+
scanner = Scanner.new(markdown)
|
|
88
|
+
text = markdown
|
|
89
|
+
|
|
90
|
+
# Built-ins each refuse to act inside a fence. Custom handlers are not
|
|
91
|
+
# short-circuited here: Context exposes #in_code_fence? precisely so a
|
|
92
|
+
# host handler can decide for itself, which it cannot do if it never runs.
|
|
93
|
+
|
|
94
|
+
BUILT_INS.each do |handler|
|
|
95
|
+
repaired = handler.new(options).call(text, scanner.context(options), scanner)
|
|
96
|
+
next if repaired == text
|
|
97
|
+
|
|
98
|
+
text = repaired
|
|
99
|
+
scanner = Scanner.new(text)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
run_custom(text, scanner.context(options))
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
private
|
|
106
|
+
def run_custom(text, context)
|
|
107
|
+
Array(options[:handlers]).reduce(text) { |current, handler| handler.call(current, context) }
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def validate_link_mode
|
|
111
|
+
return if LINK_MODES.include?(options[:link_mode])
|
|
112
|
+
|
|
113
|
+
raise UnknownOption, "link_mode must be one of #{LINK_MODES.join(", ")}"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def validate_app_tags
|
|
117
|
+
value = options[:app_tags]
|
|
118
|
+
return if !value || (value.is_a?(Array) && value.all? { |name| name.respond_to?(:to_str) || name.is_a?(Symbol) })
|
|
119
|
+
|
|
120
|
+
raise UnknownOption, "app_tags must be false or a list of tag names"
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module MaquinaRemend
|
|
4
|
+
# One pass over the buffer, tracking the only three states a handler is
|
|
5
|
+
# allowed to ask about. The guards in the pattern spec are the reason this
|
|
6
|
+
# exists: nothing may be "repaired" inside a fence, inside inline code, or
|
|
7
|
+
# inside math, and the only way to know is to have walked the buffer.
|
|
8
|
+
#
|
|
9
|
+
# A scanner is built over one immutable buffer and caches everything it
|
|
10
|
+
# derives, so the expensive walk happens once. When a handler changes the
|
|
11
|
+
# text, MaquinaRemend::Pipeline throws the scanner away and builds a new one
|
|
12
|
+
# rather than trying to update this one.
|
|
13
|
+
#
|
|
14
|
+
# Hosts should reach for MaquinaRemend.context instead: it answers the
|
|
15
|
+
# question — is the tail inside a fence, and what language is it — without
|
|
16
|
+
# exposing the scanner's internals. The scanner itself is public because the
|
|
17
|
+
# built-in handlers need it, not because it is a stable surface.
|
|
18
|
+
class Scanner
|
|
19
|
+
# A fenced code block's opening or closing line: up to three spaces of
|
|
20
|
+
# indent, three or more backticks or tildes, then an optional info string
|
|
21
|
+
# captured as group 2.
|
|
22
|
+
FENCE_LINE = /\A {0,3}(`{3,}|~{3,})[ \t]*(\S*)/
|
|
23
|
+
|
|
24
|
+
# A *balanced* inline code span. Matching only balanced spans is the point:
|
|
25
|
+
# what is left over after these are masked out is the unterminated one.
|
|
26
|
+
INLINE_CODE = /(?<!`)(`+)(?!`)(.*?)(?<!`)\1(?!`)/m
|
|
27
|
+
|
|
28
|
+
# The math delimiters whose contents must never be treated as markdown.
|
|
29
|
+
# Underscores inside `$$a_1 + b_2$$` are subscripts, not emphasis, and a
|
|
30
|
+
# LaTeX span is math even when it uses no dollar sign at all.
|
|
31
|
+
MATH_SPANS = [
|
|
32
|
+
/\$\$.*?\$\$/m, # block math
|
|
33
|
+
/\\\[.*?\\\]/m, # LaTeX display
|
|
34
|
+
/\\\(.*?\\\)/m # LaTeX inline
|
|
35
|
+
].freeze
|
|
36
|
+
|
|
37
|
+
# Masked spans keep their length but must not read as whitespace: a "**" that
|
|
38
|
+
# follows `code` is a valid closer, and blanking the code span with spaces
|
|
39
|
+
# makes it look like an opener instead. Learned from real streamed output.
|
|
40
|
+
MASK = "\u0001"
|
|
41
|
+
|
|
42
|
+
# The buffer this scanner was built over, unmodified.
|
|
43
|
+
attr_reader :text
|
|
44
|
+
|
|
45
|
+
# The info string of the fence that is currently open — `"ruby"` for a
|
|
46
|
+
# fence opened with three backticks and `ruby` — or `nil` when no fence is
|
|
47
|
+
# open or the fence carried no info string.
|
|
48
|
+
attr_reader :open_fence_info
|
|
49
|
+
|
|
50
|
+
# Walks `text` once and caches what the walk found. The buffer is not
|
|
51
|
+
# copied and never mutated.
|
|
52
|
+
def initialize(text)
|
|
53
|
+
@text = text
|
|
54
|
+
@fence_open = false
|
|
55
|
+
@open_fence_info = nil
|
|
56
|
+
@paragraph_offset = 0
|
|
57
|
+
scan
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# True when the buffer ends inside a fenced code block. Surfaces to handlers
|
|
61
|
+
# as MaquinaRemend::Context#in_code_fence?.
|
|
62
|
+
def fence_open? = @fence_open
|
|
63
|
+
|
|
64
|
+
# Everything after the last blank line outside a fence. Emphasis cannot span
|
|
65
|
+
# a blank line, so this is the only region a completion may touch.
|
|
66
|
+
def paragraph
|
|
67
|
+
text[@paragraph_offset..] || ""
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The byte offset #paragraph starts at: the end of the last blank line that
|
|
71
|
+
# was not inside a fence.
|
|
72
|
+
attr_reader :paragraph_offset
|
|
73
|
+
|
|
74
|
+
# Everything before #paragraph. A handler that rewrites the paragraph
|
|
75
|
+
# rebuilds the buffer as `scanner.prefix + repaired_paragraph`, which is how
|
|
76
|
+
# a repair stays confined to the region it is allowed to touch.
|
|
77
|
+
def prefix
|
|
78
|
+
text[0...@paragraph_offset] || ""
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# An unterminated inline code span in the paragraph: an odd number of
|
|
82
|
+
# backticks once the balanced spans are gone.
|
|
83
|
+
def inline_code_open?
|
|
84
|
+
return @inline_code_open if defined?(@inline_code_open)
|
|
85
|
+
|
|
86
|
+
@inline_code_open = masked_code.count("`").odd?
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Block math spans paragraphs, so this question is asked of the whole
|
|
90
|
+
# document - but only of the parts of it that are prose. A document that
|
|
91
|
+
# merely writes about "$$x = 1" inside a code span has no open math.
|
|
92
|
+
def math_open?
|
|
93
|
+
return @math_open if defined?(@math_open)
|
|
94
|
+
|
|
95
|
+
@math_open = masked_text.scan(/(?<!\\)\$\$/).length.odd?
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# The whole buffer with fenced blocks and inline code blanked out, lengths
|
|
99
|
+
# preserved.
|
|
100
|
+
def masked_text
|
|
101
|
+
@masked_text ||= mask_fences(text).gsub(INLINE_CODE) { MASK * ::Regexp.last_match(0).length }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The paragraph with inline code and math replaced by placeholders of the
|
|
105
|
+
# same length, so delimiter counting cannot see inside them.
|
|
106
|
+
def masked_paragraph
|
|
107
|
+
@masked_paragraph ||= MATH_SPANS.reduce(masked_code) do |masked, pattern|
|
|
108
|
+
masked.gsub(pattern) { MASK * ::Regexp.last_match(0).length }
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# The MaquinaRemend::Context for this buffer, carrying `options` through to
|
|
113
|
+
# any handler that wants to read them. Memoised per option hash, since the
|
|
114
|
+
# pipeline asks for the same one repeatedly.
|
|
115
|
+
def context(options = {})
|
|
116
|
+
@context ||= {}
|
|
117
|
+
@context[options] ||=
|
|
118
|
+
Context.new(
|
|
119
|
+
in_code_fence: fence_open?,
|
|
120
|
+
in_inline_code: inline_code_open?,
|
|
121
|
+
in_math: math_open?,
|
|
122
|
+
open_fence_info: open_fence_info,
|
|
123
|
+
options: options
|
|
124
|
+
)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
private
|
|
128
|
+
def mask_fences(source)
|
|
129
|
+
inside = false
|
|
130
|
+
marker = nil
|
|
131
|
+
|
|
132
|
+
source.lines.map { |line|
|
|
133
|
+
match = FENCE_LINE.match(line.chomp)
|
|
134
|
+
|
|
135
|
+
if match && inside && closes?(marker, match[1])
|
|
136
|
+
inside = false
|
|
137
|
+
marker = nil
|
|
138
|
+
MASK * line.length
|
|
139
|
+
elsif match && !inside
|
|
140
|
+
inside = true
|
|
141
|
+
marker = match[1]
|
|
142
|
+
MASK * line.length
|
|
143
|
+
elsif inside
|
|
144
|
+
MASK * line.length
|
|
145
|
+
else
|
|
146
|
+
line
|
|
147
|
+
end
|
|
148
|
+
}.join
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def masked_code
|
|
152
|
+
@masked_code ||= paragraph.gsub(INLINE_CODE) { MASK * ::Regexp.last_match(0).length }
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def scan
|
|
156
|
+
offset = 0
|
|
157
|
+
marker = nil
|
|
158
|
+
|
|
159
|
+
text.each_line do |line|
|
|
160
|
+
stripped = line.chomp
|
|
161
|
+
|
|
162
|
+
if (match = FENCE_LINE.match(stripped))
|
|
163
|
+
if @fence_open && closes?(marker, match[1])
|
|
164
|
+
@fence_open = false
|
|
165
|
+
@open_fence_info = nil
|
|
166
|
+
marker = nil
|
|
167
|
+
@paragraph_offset = offset + line.length
|
|
168
|
+
elsif !@fence_open
|
|
169
|
+
@fence_open = true
|
|
170
|
+
marker = match[1]
|
|
171
|
+
@open_fence_info = match[2].empty? ? nil : match[2]
|
|
172
|
+
end
|
|
173
|
+
elsif !@fence_open && stripped.strip.empty?
|
|
174
|
+
@paragraph_offset = offset + line.length
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
offset += line.length
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def closes?(opening, candidate)
|
|
182
|
+
return false unless opening
|
|
183
|
+
|
|
184
|
+
opening[0] == candidate[0] && candidate.length >= opening.length
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module MaquinaRemend
|
|
4
|
+
# The released version of the gem, as a `String`. Follows semantic
|
|
5
|
+
# versioning; the option names in MaquinaRemend::DEFAULTS and the handler
|
|
6
|
+
# contract are the public surface a major bump would be about.
|
|
7
|
+
VERSION = "0.1.0"
|
|
8
|
+
end
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "maquina_remend/version"
|
|
4
|
+
require_relative "maquina_remend/context"
|
|
5
|
+
require_relative "maquina_remend/scanner"
|
|
6
|
+
require_relative "maquina_remend/pipeline"
|
|
7
|
+
|
|
8
|
+
# Repairs the tail of a markdown buffer that is still being streamed, so that a
|
|
9
|
+
# renderer never sees half a bold run or a link whose URL has not arrived yet.
|
|
10
|
+
#
|
|
11
|
+
# ```ruby
|
|
12
|
+
# MaquinaRemend.call("**unclosed bold") # => "**unclosed bold**"
|
|
13
|
+
# MaquinaRemend.call("`code") # => "`code`"
|
|
14
|
+
# MaquinaRemend.call("[label](https://exa") # => "[label](#)"
|
|
15
|
+
# ```
|
|
16
|
+
#
|
|
17
|
+
# Pure function. Idempotent. Returns well-formed input unchanged, byte for byte.
|
|
18
|
+
# Zero runtime dependencies, and no Rails: this gem must be usable from a plain
|
|
19
|
+
# Ruby process with nothing else loaded.
|
|
20
|
+
#
|
|
21
|
+
# ## The shape of the library
|
|
22
|
+
#
|
|
23
|
+
# | Object | Role |
|
|
24
|
+
# |---|---|
|
|
25
|
+
# | MaquinaRemend.call | the whole public API: buffer in, repaired buffer out |
|
|
26
|
+
# | MaquinaRemend.context | the scanner's reading of a buffer, with no repair |
|
|
27
|
+
# | MaquinaRemend::DEFAULTS | every option and its default |
|
|
28
|
+
# | MaquinaRemend::Context | what a handler is allowed to know |
|
|
29
|
+
# | MaquinaRemend::Pipeline | the fixed order the built-in handlers run in |
|
|
30
|
+
# | MaquinaRemend::Handlers | one class per repair |
|
|
31
|
+
#
|
|
32
|
+
# ## Guarantees
|
|
33
|
+
#
|
|
34
|
+
# Three properties are asserted over the whole test corpus rather than over
|
|
35
|
+
# hand-picked inputs, and every handler is written to keep them:
|
|
36
|
+
#
|
|
37
|
+
# 1. **Idempotence** — `call(call(x)) == call(x)`.
|
|
38
|
+
# 2. **No-op on well-formed input** — a complete document comes back
|
|
39
|
+
# byte-identical.
|
|
40
|
+
# 3. **Prefix safety** — every truncation point of a document is repairable
|
|
41
|
+
# without raising.
|
|
42
|
+
#
|
|
43
|
+
# A repair that cannot be made safe is not made. Guards matter more than
|
|
44
|
+
# completions here: a false repair corrupts a message that was never broken.
|
|
45
|
+
# Nothing is ever completed inside a fenced code block, inside an inline code
|
|
46
|
+
# span, or inside math, because in those places broken-looking markup is the
|
|
47
|
+
# content.
|
|
48
|
+
module MaquinaRemend
|
|
49
|
+
# The application tag names closed by default: the tags a model emits to
|
|
50
|
+
# carry application meaning rather than presentation. Each host gives them
|
|
51
|
+
# meaning of its own, so `app_tags:` takes a list rather than a boolean; this
|
|
52
|
+
# is the list a host gets when it says nothing.
|
|
53
|
+
APP_TAGS = %w[thinking answer tool_call citation scratchpad].freeze
|
|
54
|
+
|
|
55
|
+
# Every repair is individually disableable. Inline math is off because a bare
|
|
56
|
+
# dollar sign is far more often currency than mathematics.
|
|
57
|
+
#
|
|
58
|
+
# Pass any subset of these as keyword arguments to MaquinaRemend.call; an
|
|
59
|
+
# unrecognised key raises MaquinaRemend::UnknownOption rather than being
|
|
60
|
+
# ignored, so a typo in a host's configuration surfaces at once.
|
|
61
|
+
#
|
|
62
|
+
# ### Completion handlers
|
|
63
|
+
#
|
|
64
|
+
# Each flag switches one repair on or off.
|
|
65
|
+
#
|
|
66
|
+
# | Option | Default | Malformed input | Repaired to | Handler |
|
|
67
|
+
# |---|---|---|---|---|
|
|
68
|
+
# | `bold:` | `true` | `**unclosed bold` | `**unclosed bold**` | Handlers::Emphasis |
|
|
69
|
+
# | `italic:` | `true` | `*unclosed` / `_unclosed` | `*unclosed*` / `_unclosed_` | Handlers::Emphasis |
|
|
70
|
+
# | `bold_italic:` | `true` | `***both` | `***both***` | Handlers::Emphasis |
|
|
71
|
+
# | `inline_code:` | `true` | `` `code `` | `` `code` `` | Handlers::InlineCode |
|
|
72
|
+
# | `strikethrough:` | `true` | `~~struck` | `~~struck~~` | Handlers::Strikethrough |
|
|
73
|
+
# | `links:` | `true` | `[label` | `[label]()` | Handlers::Links |
|
|
74
|
+
# | `images:` | `true` | `![alt` | `![alt]()` | Handlers::Links |
|
|
75
|
+
# | `block_math:` | `true` | `$$x = 1` | `$$x = 1$$` | Handlers::Math |
|
|
76
|
+
# | `inline_math:` | **`false`** | `$x = 1` | `$x = 1$` | Handlers::Math |
|
|
77
|
+
# | `setext_headings:` | `true` | `Title` then a lone `=` | underline padded to the width of the title | Handlers::SetextHeading |
|
|
78
|
+
# | `comparison_operators:` | `true` | `- a > b` | `- a \> b` | Handlers::ComparisonOperators |
|
|
79
|
+
# | `html_tags:` | `true` | `text <div cla` | `text` — the truncated tag is dropped | Handlers::HtmlTags |
|
|
80
|
+
# | `single_tilde:` | `true` | `20~25°C` | `20\~25°C` | Handlers::SingleTilde |
|
|
81
|
+
# | `dangling_escape:` | `true` | a buffer ending in a lone `\` | the trailing backslash is dropped | Handlers::DanglingEscape |
|
|
82
|
+
# | `app_tags:` | MaquinaRemend::APP_TAGS | `<thinking>` with no closer yet | `</thinking>` appended | Handlers::AppTags |
|
|
83
|
+
#
|
|
84
|
+
# **`inline_math:` is off because a bare dollar sign is currency far more
|
|
85
|
+
# often than it is mathematics.** `costs $5 and $10` would otherwise acquire a
|
|
86
|
+
# closing `$` and turn a price list into an equation. Turn it on only if you
|
|
87
|
+
# know your corpus.
|
|
88
|
+
#
|
|
89
|
+
# ### `app_tags:`
|
|
90
|
+
#
|
|
91
|
+
# The list of application tag names MaquinaRemend::Handlers::AppTags will
|
|
92
|
+
# close, MaquinaRemend::APP_TAGS by default. Pass your own list to match the
|
|
93
|
+
# host's tag registry, or `false` to switch the handler off. Anything that is
|
|
94
|
+
# neither a list of names nor `false` raises MaquinaRemend::UnknownOption.
|
|
95
|
+
#
|
|
96
|
+
# ```ruby
|
|
97
|
+
# MaquinaRemend.call(buffer, app_tags: %w[thinking answer plan])
|
|
98
|
+
# MaquinaRemend.call(buffer, app_tags: false)
|
|
99
|
+
# ```
|
|
100
|
+
#
|
|
101
|
+
# **Only the listed names are closed, and the list is short on purpose.**
|
|
102
|
+
# Closing every unbalanced tag would close the ones that must not be closed:
|
|
103
|
+
# `<br>` and `<img>` have no closer, and a `<div>` wrapper is routinely opened
|
|
104
|
+
# in one streamed block and closed several blocks later, so a speculative
|
|
105
|
+
# `</div>` would land in the middle of its own content. An application tag is
|
|
106
|
+
# different in kind — it is emitted whole by the model, it wraps a single
|
|
107
|
+
# region of the answer, and the host knows its name in advance.
|
|
108
|
+
#
|
|
109
|
+
# ### `link_mode:`
|
|
110
|
+
#
|
|
111
|
+
# `:protocol` (the default) or `:text_only`. Anything else raises
|
|
112
|
+
# MaquinaRemend::UnknownOption. See MaquinaRemend::LINK_MODES.
|
|
113
|
+
#
|
|
114
|
+
# | Value | `[label` | `[label](https://exa` |
|
|
115
|
+
# |---|---|---|
|
|
116
|
+
# | `:protocol` | `[label]()` | `[label](#)` |
|
|
117
|
+
# | `:text_only` | `label` | `label` |
|
|
118
|
+
#
|
|
119
|
+
# A half-streamed destination is **never** kept. `https://exa` is a perfectly
|
|
120
|
+
# valid URL to the wrong host, and model output is untrusted; the destination
|
|
121
|
+
# is replaced by a placeholder until the real one arrives.
|
|
122
|
+
#
|
|
123
|
+
# ### `handlers:`
|
|
124
|
+
#
|
|
125
|
+
# An array of custom handlers, run after the built-ins. MaquinaRemend.call
|
|
126
|
+
# documents the contract.
|
|
127
|
+
DEFAULTS = {
|
|
128
|
+
bold: true,
|
|
129
|
+
italic: true,
|
|
130
|
+
bold_italic: true,
|
|
131
|
+
inline_code: true,
|
|
132
|
+
strikethrough: true,
|
|
133
|
+
links: true,
|
|
134
|
+
images: true,
|
|
135
|
+
block_math: true,
|
|
136
|
+
inline_math: false,
|
|
137
|
+
setext_headings: true,
|
|
138
|
+
comparison_operators: true,
|
|
139
|
+
html_tags: true,
|
|
140
|
+
single_tilde: true,
|
|
141
|
+
dangling_escape: true,
|
|
142
|
+
app_tags: APP_TAGS,
|
|
143
|
+
link_mode: :protocol,
|
|
144
|
+
handlers: []
|
|
145
|
+
}.freeze
|
|
146
|
+
|
|
147
|
+
# The accepted values for the `link_mode:` option: `:protocol` and
|
|
148
|
+
# `:text_only`. Both are described under MaquinaRemend::DEFAULTS.
|
|
149
|
+
LINK_MODES = %i[protocol text_only].freeze
|
|
150
|
+
|
|
151
|
+
# Base class for everything this gem raises, so a host can rescue the library
|
|
152
|
+
# as a whole without naming each error.
|
|
153
|
+
class Error < StandardError; end
|
|
154
|
+
|
|
155
|
+
# Raised when MaquinaRemend.call is given an option key that is not in
|
|
156
|
+
# MaquinaRemend::DEFAULTS, or a `link_mode:` outside
|
|
157
|
+
# MaquinaRemend::LINK_MODES.
|
|
158
|
+
#
|
|
159
|
+
# Unknown options are an error rather than a silent no-op: a misspelled flag
|
|
160
|
+
# would otherwise leave a repair quietly enabled that the host believed it had
|
|
161
|
+
# turned off.
|
|
162
|
+
class UnknownOption < Error; end
|
|
163
|
+
|
|
164
|
+
# Repairs the tail of `markdown` and returns the result.
|
|
165
|
+
#
|
|
166
|
+
# ```ruby
|
|
167
|
+
# MaquinaRemend.call(markdown, **options) # => String
|
|
168
|
+
# ```
|
|
169
|
+
#
|
|
170
|
+
# * `markdown` — the buffer so far, including its incomplete tail. `nil` and
|
|
171
|
+
# the empty string are returned exactly as they came in.
|
|
172
|
+
# * `options` — any subset of MaquinaRemend::DEFAULTS.
|
|
173
|
+
# * Returns a `String` a markdown parser can render without producing
|
|
174
|
+
# half-open constructs.
|
|
175
|
+
# * Raises MaquinaRemend::UnknownOption for an option key outside
|
|
176
|
+
# MaquinaRemend::DEFAULTS, or a `link_mode:` outside
|
|
177
|
+
# MaquinaRemend::LINK_MODES.
|
|
178
|
+
#
|
|
179
|
+
# The buffer is scanned once, passed through the built-in handlers in
|
|
180
|
+
# MaquinaRemend::Pipeline::BUILT_INS order, then through any handlers given in
|
|
181
|
+
# `handlers:`.
|
|
182
|
+
#
|
|
183
|
+
# ```ruby
|
|
184
|
+
# MaquinaRemend.call("A **bold run that is still")
|
|
185
|
+
# # => "A **bold run that is still**"
|
|
186
|
+
#
|
|
187
|
+
# MaquinaRemend.call("costs $5", inline_math: true)
|
|
188
|
+
# # => "costs $5$" -- which is why inline_math is off by default
|
|
189
|
+
#
|
|
190
|
+
# MaquinaRemend.call("[label](https://exa", link_mode: :text_only)
|
|
191
|
+
# # => "label"
|
|
192
|
+
# ```
|
|
193
|
+
#
|
|
194
|
+
# ## Custom handlers
|
|
195
|
+
#
|
|
196
|
+
# A handler is any object that responds to `#call(text, context)` and returns
|
|
197
|
+
# the buffer — repaired, or untouched. There is no class to inherit from and
|
|
198
|
+
# no registration step.
|
|
199
|
+
#
|
|
200
|
+
# ```ruby
|
|
201
|
+
# class MyHandler
|
|
202
|
+
# def call(text, context) # context: MaquinaRemend::Context
|
|
203
|
+
# return text if context.in_code_fence?
|
|
204
|
+
#
|
|
205
|
+
# text.end_with?("::") ? text + " " : text
|
|
206
|
+
# end
|
|
207
|
+
# end
|
|
208
|
+
#
|
|
209
|
+
# MaquinaRemend.call(markdown, handlers: [MyHandler.new])
|
|
210
|
+
# ```
|
|
211
|
+
#
|
|
212
|
+
# The contract, in full:
|
|
213
|
+
#
|
|
214
|
+
# * **Arity is two.** `text` is the buffer as the built-ins left it; `context`
|
|
215
|
+
# is a MaquinaRemend::Context describing where the tail sits.
|
|
216
|
+
# * **Return a String.** Returning the argument unchanged is the normal
|
|
217
|
+
# outcome; most passes over most buffers repair nothing.
|
|
218
|
+
# * **Custom handlers run last**, in the order given, each one seeing the
|
|
219
|
+
# previous one's output.
|
|
220
|
+
# * **Unlike the built-ins, a custom handler still runs inside a fenced code
|
|
221
|
+
# block.** It is given the context and trusted to decide, which it could not
|
|
222
|
+
# do if the pipeline had already returned. Ask
|
|
223
|
+
# MaquinaRemend::Context#in_code_fence? and return early if a fence should
|
|
224
|
+
# stop you.
|
|
225
|
+
# * **Stay idempotent, and stay byte-identical on well-formed input.** Those
|
|
226
|
+
# two properties belong to the library, not just to the built-ins, and a
|
|
227
|
+
# handler that appends unconditionally breaks both — the text it appended
|
|
228
|
+
# dangles on the next pass and gets appended to again.
|
|
229
|
+
#
|
|
230
|
+
# A handler is not given the MaquinaRemend::Scanner. Context is deliberately
|
|
231
|
+
# the whole of what a handler may know; a repair that needs more state than it
|
|
232
|
+
# exposes is a sign the scanner is missing a state, not a sign that the
|
|
233
|
+
# context object should grow.
|
|
234
|
+
def self.call(markdown, **options)
|
|
235
|
+
Pipeline.new(**options).call(markdown)
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# The scanner's view of a buffer, without repairing it. Callers that need to
|
|
239
|
+
# know whether the tail sits inside a fence - a renderer deciding whether to
|
|
240
|
+
# highlight, for one - should ask this rather than reach into Scanner.
|
|
241
|
+
#
|
|
242
|
+
# ```ruby
|
|
243
|
+
# context = MaquinaRemend.context("~~~ruby\ndef call")
|
|
244
|
+
# context.in_code_fence? # => true
|
|
245
|
+
# context.open_fence_info # => "ruby"
|
|
246
|
+
# ```
|
|
247
|
+
#
|
|
248
|
+
# * `markdown` — the buffer so far; coerced with `#to_s`, so `nil` is a valid
|
|
249
|
+
# argument and reads as an empty document.
|
|
250
|
+
# * Returns a MaquinaRemend::Context: the three states a handler is allowed to
|
|
251
|
+
# ask about, plus the open fence's info string.
|
|
252
|
+
def self.context(markdown)
|
|
253
|
+
Scanner.new(markdown.to_s).context
|
|
254
|
+
end
|
|
255
|
+
end
|