streaming-markdown-parser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +128 -0
- data/lib/streaming_markdown_parser/closer.rb +110 -0
- data/lib/streaming_markdown_parser/listener.rb +36 -0
- data/lib/streaming_markdown_parser/repair.rb +25 -0
- data/lib/streaming_markdown_parser/version.rb +6 -0
- data/lib/streaming_markdown_parser.rb +264 -0
- data/sig/streaming_markdown_parser.rbs +334 -0
- metadata +75 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 260c22579b83578d52e44d3f3af59882ed416e9dd244adca19218e501034ae2c
|
|
4
|
+
data.tar.gz: 3b27337ac49d3f75936e7ee5bafbbe52cb2f80541a3629fe4b4937805c5adc10
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: e20bbf849e2ca61f6819a2bde74c7424e87cf7fff73eb61b37b96a5781ae19e0b89c9b9a4f832b3cddcc9485fb7a6fd8dbb673eaae9db5fb524027cc055cced9
|
|
7
|
+
data.tar.gz: bd6ea2565009925fb8e9c721b34d2dde0f26aa10bfdb4094403184cd0dc0b56db9e24684f96c8503268dc42af689ec19bd3b30882e32ae871552387b1bced821
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Martin Vysny
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# streaming-markdown-parser
|
|
2
|
+
|
|
3
|
+
Parses Markdown as it streams in, an LLM's reply delta by delta, into top-level kramdown blocks.
|
|
4
|
+
Each finished block reaches your listener once and never changes; the unfinished tail comes
|
|
5
|
+
again, provisional, on every delta. It draws nothing: the terminal, the web page or the window
|
|
6
|
+
keeps its own kramdown converter.
|
|
7
|
+
|
|
8
|
+
## Install
|
|
9
|
+
|
|
10
|
+
```ruby
|
|
11
|
+
gem 'streaming-markdown-parser', '~> 0.1.0', require: 'streaming_markdown_parser'
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Before 1.0 a minor version may break the API, hence the `~> 0.1.0`. Outside Bundler,
|
|
15
|
+
`gem install streaming-markdown-parser`, then `require 'streaming_markdown_parser'`.
|
|
16
|
+
|
|
17
|
+
Ruby 3.3 or newer; the only dependencies are `kramdown` and `kramdown-parser-gfm`.
|
|
18
|
+
|
|
19
|
+
## Use
|
|
20
|
+
|
|
21
|
+
A listener includes `StreamingMarkdownParser::Listener` and overrides what it needs. Every call
|
|
22
|
+
gets a kramdown root, which any converter takes (but see *Rendering block by block*):
|
|
23
|
+
|
|
24
|
+
```ruby
|
|
25
|
+
class HtmlView
|
|
26
|
+
include StreamingMarkdownParser::Listener
|
|
27
|
+
|
|
28
|
+
def initialize
|
|
29
|
+
@done = +''
|
|
30
|
+
@tail = ''
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def on_block(root) = @done << Kramdown::Converter::Html.convert(root).first
|
|
34
|
+
def on_open(root) = @tail = Kramdown::Converter::Html.convert(root).first
|
|
35
|
+
def on_reset = @done.clear
|
|
36
|
+
def to_html = @done + @tail
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
parser = StreamingMarkdownParser.new(view = HtmlView.new)
|
|
40
|
+
parser.append("Hello.\n\nWor") # on_open: "Hello.", "Wor" (the next line may continue "Hello.")
|
|
41
|
+
parser.append("ld.\n") # on_block: "Hello."; on_open: "World."
|
|
42
|
+
parser.finish # on_block: "World."; on_open: empty (no text given: no on_reset)
|
|
43
|
+
view.to_html # => "<p>Hello.</p>\n<p>World.</p>\n"
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
| Call | When | Final? |
|
|
47
|
+
|---|---|---|
|
|
48
|
+
| `on_block(root)` | a top-level block is finished: a paragraph, a list, a table, a fence | yes, sent once |
|
|
49
|
+
| `on_open(root)` | after every delta: the tail after the last finished block | no, each replaces the last |
|
|
50
|
+
| `on_reset` | `finish(text)` got a final text that differs from the deltas | drop every block of this reply; the final text's blocks follow, from the first |
|
|
51
|
+
|
|
52
|
+
A listener that wants only final output implements `on_block` alone.
|
|
53
|
+
|
|
54
|
+
One parser and one listener per reply: after `finish`, `append` and another `finish` raise, and
|
|
55
|
+
`on_reset` drops every block the listener holds, so a listener shared with an earlier reply would
|
|
56
|
+
lose that reply too.
|
|
57
|
+
|
|
58
|
+
A block is one top-level element of the reply: a paragraph, a heading, a table, a fence, a rule,
|
|
59
|
+
or a whole list or blockquote with everything inside it. In prose a block *is* a paragraph: each
|
|
60
|
+
`on_block` is one finished paragraph, and the next one is in `on_open`.
|
|
61
|
+
|
|
62
|
+
Once the stream ends, render the reply once more from its whole text, as only there does every
|
|
63
|
+
reference link and footnote resolve (see *When is a block finished?*). The same goes for a reply
|
|
64
|
+
that arrives whole (a non-streaming provider, a replayed transcript):
|
|
65
|
+
|
|
66
|
+
```ruby
|
|
67
|
+
root = StreamingMarkdownParser.parse(text) # one root, for one converter call
|
|
68
|
+
StreamingMarkdownParser.new(view).finish(text) # or the same calls a stream makes, no on_reset
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Take a fresh parser for that: the streaming one's `finish(text)`, given the text it streamed,
|
|
72
|
+
keeps the blocks it sent. And convert `parse`'s root in one call, or each block gets its own
|
|
73
|
+
footnotes list.
|
|
74
|
+
|
|
75
|
+
### When the final text differs
|
|
76
|
+
|
|
77
|
+
Some providers stream a preview, and the final message they hand over at the end is not quite
|
|
78
|
+
the deltas added up. Pass it to `finish(text)`:
|
|
79
|
+
|
|
80
|
+
- **The same text as the deltas:** an ordinary finish. The open tail becomes blocks, and the
|
|
81
|
+
blocks already sent stay.
|
|
82
|
+
- **A different text:** the change may be anywhere, even in the first block, so no block sent
|
|
83
|
+
so far can be trusted. The parser starts over, all within that `finish` call: `on_reset`,
|
|
84
|
+
then `on_block` for every block of the final text from the first, then an empty `on_open`.
|
|
85
|
+
The listener drops what it built for this reply on `on_reset` and builds it again from the
|
|
86
|
+
calls that follow, as `HtmlView` above does.
|
|
87
|
+
|
|
88
|
+
`finish` with no text takes the deltas as they came and never resets. Neither does a parser
|
|
89
|
+
that got no delta before `finish(text)`, as there is nothing to drop.
|
|
90
|
+
|
|
91
|
+
### Rendering block by block
|
|
92
|
+
|
|
93
|
+
Two things a whole-document render gets for free are the listener's job:
|
|
94
|
+
|
|
95
|
+
- **Spacing between blocks.** Blocks stack vertically, each below the last at full width, never
|
|
96
|
+
side by side, and with no separator but a rule the reply itself has. The source's blank lines
|
|
97
|
+
are dropped, so the listener puts its own spacing between them. HTML needs none, as `<p>`
|
|
98
|
+
brings its own margin; a terminal or plain-text converter does.
|
|
99
|
+
- **State from block to block.** Each root is converted alone, so a converter that remembers
|
|
100
|
+
something across blocks needs the listener to carry it: the indent under the last header,
|
|
101
|
+
footnote numbers. Such an app takes a whole reply through `new(listener).finish(text)` too, not
|
|
102
|
+
`.parse`, so a reply renders alike however it arrived.
|
|
103
|
+
- **Footnotes.** A `:footnote` span holds its label in `options[:name]` and its definition, a
|
|
104
|
+
`:footnote_def` element, in `value`: not a String, so a converter's catch-all that prints
|
|
105
|
+
`value` prints the element. The definition block itself is gone from the tree, and the number
|
|
106
|
+
is the converter's: kramdown's HTML converter numbers each root from 1 and writes the endnotes
|
|
107
|
+
after it, so block by block the numbers restart at 1 and each block repeats its own endnotes.
|
|
108
|
+
|
|
109
|
+
## When is a block finished?
|
|
110
|
+
|
|
111
|
+
When the next one has begun, as kramdown sees it. After a blank line, or a closing fence, the
|
|
112
|
+
first complete line is parsed with the open text before it, and alone. Only if the blocks come
|
|
113
|
+
out the same is the cut taken, so a loose list, an indented continuation or a fenced block inside
|
|
114
|
+
a list item stays open until it truly ends. However the reply is cut into deltas, the blocks equal
|
|
115
|
+
kramdown's parse of the whole text, references aside.
|
|
116
|
+
|
|
117
|
+
A reference link (`[the docs][1]`) or footnote (`[^a]`) resolves against a definition anywhere in
|
|
118
|
+
the text, but each side of a cut is parsed alone. A definition right before or after the block
|
|
119
|
+
stays with it; one a block or more away leaves the reference literal, and the definition shows as
|
|
120
|
+
nothing. A model that cites its sources at the end streams `[the docs][1]` and no URL. The parser
|
|
121
|
+
never takes such a block back: the render of the whole reply at the end resolves it.
|
|
122
|
+
|
|
123
|
+
Before parsing, `StreamingMarkdownParser::Repair` fixes what LLMs get wrong; today, a table
|
|
124
|
+
separator written with em-dashes (`|—-|`), which kramdown would not take for a table.
|
|
125
|
+
|
|
126
|
+
## License
|
|
127
|
+
|
|
128
|
+
MIT
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class StreamingMarkdownParser
|
|
4
|
+
# The blocks that only their closer ends: fences, `$$` math, HTML comments and kramdown's
|
|
5
|
+
# extensions. Until the closer comes, kramdown reads paragraphs, so the parser takes no cut
|
|
6
|
+
# inside one. It holds the open block's closer and hands it each line, getting back the closer
|
|
7
|
+
# for the next:
|
|
8
|
+
#
|
|
9
|
+
# closer, rest = Closer.open("$$ a = b\n", 9) # a Closer::Math; rest is " a = b\n"
|
|
10
|
+
# closer = closer.watch(rest) # itself: still open
|
|
11
|
+
# closer = closer.watch("c = d $$\n") # a Closer::MathKept: math if the next line agrees
|
|
12
|
+
# closer.watch("next\n") # Closer::Rewind[to: 9]: kramdown reverts it
|
|
13
|
+
#
|
|
14
|
+
# `watch` answers itself while the block is open, nil once it closed on that line, {PASS} when
|
|
15
|
+
# the line comes after the block and is scanned like any other, or a {Rewind} when kramdown
|
|
16
|
+
# reverts the block to paragraphs, so a fence inside it counts after all.
|
|
17
|
+
#
|
|
18
|
+
# Each closer is kramdown's own; one too loose takes a cut that the real closer then undoes.
|
|
19
|
+
module Closer
|
|
20
|
+
# The line is past the block; scan it as any other.
|
|
21
|
+
# @return [Symbol]
|
|
22
|
+
PASS = :pass
|
|
23
|
+
|
|
24
|
+
# @return [Regexp] an extension kramdown knows, alone on its line; the self-closing
|
|
25
|
+
# `{::comment /}` opens nothing
|
|
26
|
+
EXTENSION = %r{\A {0,3}\{::(comment|nomarkdown|options)(?:\s.*?)?(?<!/)\}\s*\z}
|
|
27
|
+
|
|
28
|
+
# @return [Regexp] the line kramdown needs right after a `$$` block's closer to keep it math:
|
|
29
|
+
# blank, an IAL or `^`
|
|
30
|
+
MATH_KEPT = /\A(?:#{Kramdown::Parser::Kramdown::BLOCK_BOUNDARY})/
|
|
31
|
+
|
|
32
|
+
# @param line [String] a line where a block may start
|
|
33
|
+
# @param body [Integer] where the line after it starts in the source
|
|
34
|
+
# @return [Array(Object, String), nil] the closer `line` opens, if any, and the rest of `line`
|
|
35
|
+
# after the opener, for that closer's first {#watch}
|
|
36
|
+
def self.open(line, body)
|
|
37
|
+
if (fence = line[/\A\s*(`{3,}|~{3,})/, 1])
|
|
38
|
+
[Line.new(/\A\s*#{Regexp.escape(fence[0])}{#{fence.size},}\s*\z/, nil), '']
|
|
39
|
+
elsif (name = line[EXTENSION, 1])
|
|
40
|
+
[Line.new(%r{\A {0,3}\{:/(?:#{name})?\}\s*\z}, "#{line.chomp}\n{:/}\n"), '']
|
|
41
|
+
elsif (opener = line[/\A {0,3}<!--/])
|
|
42
|
+
[Comment.new(opener), line[opener.size..]]
|
|
43
|
+
elsif (opener = line[/\A {0,3}\$\$/])
|
|
44
|
+
[Math.new(opener, body), line[opener.size..]]
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Ends at a whole line: a fence of the same character, at least as long, or an extension's
|
|
49
|
+
# `{:/name}` or `{:/}`.
|
|
50
|
+
#
|
|
51
|
+
# @!attribute [r] pattern
|
|
52
|
+
# @return [Regexp] the closing line
|
|
53
|
+
# @!attribute [r] closed_opener
|
|
54
|
+
# @return [String, nil] the opener closed on the spot, for kramdown to confirm it opens a
|
|
55
|
+
# block; nil when no check is needed
|
|
56
|
+
Line = Data.define(:pattern, :closed_opener) do
|
|
57
|
+
# @param line [String]
|
|
58
|
+
# @return [Object, nil] see {Closer}
|
|
59
|
+
def watch(line) = pattern.match?(line) ? nil : self
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Ends at the first `-->`, mid-line too. kramdown checks nothing after it.
|
|
63
|
+
#
|
|
64
|
+
# @!attribute [r] opener
|
|
65
|
+
# @return [String] `<!--` and the spaces before it
|
|
66
|
+
Comment = Data.define(:opener) do
|
|
67
|
+
# @return [String] see {Line#closed_opener}
|
|
68
|
+
def closed_opener = "#{opener}-->\n"
|
|
69
|
+
|
|
70
|
+
# @param line [String]
|
|
71
|
+
# @return [Object, nil] see {Closer}
|
|
72
|
+
def watch(line) = line.include?('-->') ? nil : self
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Ends at the first `$$`, mid-line too; text after it on its line reverts the block.
|
|
76
|
+
#
|
|
77
|
+
# @!attribute [r] opener
|
|
78
|
+
# @return [String] `$$` and the spaces before it
|
|
79
|
+
# @!attribute [r] body
|
|
80
|
+
# @return [Integer] where the line after the opener starts, to rescan from
|
|
81
|
+
Math = Data.define(:opener, :body) do
|
|
82
|
+
# @return [String] see {Line#closed_opener}
|
|
83
|
+
def closed_opener = "#{opener}$$\n"
|
|
84
|
+
|
|
85
|
+
# @param line [String]
|
|
86
|
+
# @return [Object, nil] see {Closer}
|
|
87
|
+
def watch(line)
|
|
88
|
+
return self unless (at = line.index('$$'))
|
|
89
|
+
|
|
90
|
+
line[(at + 2)..].strip.empty? ? MathKept.new(body) : Rewind.new(body)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Judges the line after a `$$` block's closer: math only if it is blank, an IAL or `^`.
|
|
95
|
+
#
|
|
96
|
+
# @!attribute [r] body
|
|
97
|
+
# @return [Integer] see {Math#body}
|
|
98
|
+
MathKept = Data.define(:body) do
|
|
99
|
+
# @param line [String]
|
|
100
|
+
# @return [Object] see {Closer}
|
|
101
|
+
def watch(line) = MATH_KEPT.match?(line) ? PASS : Rewind.new(body)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# kramdown reverted the block: scan again from `to`, with no block open.
|
|
105
|
+
#
|
|
106
|
+
# @!attribute [r] to
|
|
107
|
+
# @return [Integer] where in the source to scan from
|
|
108
|
+
Rewind = Data.define(:to)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class StreamingMarkdownParser
|
|
4
|
+
# What a {StreamingMarkdownParser} calls. Include it and override what you use; the rest do
|
|
5
|
+
# nothing. Each call gets a kramdown root, which a converter takes:
|
|
6
|
+
#
|
|
7
|
+
# def on_block(root) = @html << Kramdown::Converter::Html.convert(root).first
|
|
8
|
+
#
|
|
9
|
+
# Calls come on the thread that called {StreamingMarkdownParser#append} or
|
|
10
|
+
# {StreamingMarkdownParser#finish}.
|
|
11
|
+
#
|
|
12
|
+
# Don't raise. A raise propagates out of that `append` or `finish`, and the calls it still owed
|
|
13
|
+
# are lost, not sent again: the parser goes on, but the listener's copy of the reply has a gap.
|
|
14
|
+
module Listener
|
|
15
|
+
# One finished top-level block (a paragraph, a heading, a whole list), a root with that block
|
|
16
|
+
# as its only child. Final: it is never called back or changed, except by {#on_reset}.
|
|
17
|
+
#
|
|
18
|
+
# @param root [Kramdown::Element]
|
|
19
|
+
# @return [void]
|
|
20
|
+
def on_block(root); end
|
|
21
|
+
|
|
22
|
+
# The unfinished tail after the last finished block, parsed as it stands: zero or more
|
|
23
|
+
# blocks, provisional. Each call replaces the one before; after the stream ends, it comes
|
|
24
|
+
# empty.
|
|
25
|
+
#
|
|
26
|
+
# @param root [Kramdown::Element]
|
|
27
|
+
# @return [void]
|
|
28
|
+
def on_open(root); end
|
|
29
|
+
|
|
30
|
+
# The final text differed from the streamed one: forget every block so far, all of them come
|
|
31
|
+
# again.
|
|
32
|
+
#
|
|
33
|
+
# @return [void]
|
|
34
|
+
def on_reset; end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class StreamingMarkdownParser
|
|
4
|
+
# Fixes what LLMs get wrong in Markdown before kramdown parses it. Today one thing: a table
|
|
5
|
+
# separator row written with typographic dashes, which kramdown won't take, so the table falls
|
|
6
|
+
# apart into paragraphs of literal pipes.
|
|
7
|
+
#
|
|
8
|
+
# Repair.call("| a | b |\n|—-|—-|\n| 1 | 2 |") # => "| a | b |\n|--|--|\n| 1 | 2 |"
|
|
9
|
+
#
|
|
10
|
+
# Only a line made wholly of separator characters with an inner pipe is touched, so prose with
|
|
11
|
+
# an em-dash and a `---` rule stay as they are.
|
|
12
|
+
module Repair
|
|
13
|
+
# @return [Regexp] a line of only pipes, dashes (ASCII, en, em), colons and spaces, with at
|
|
14
|
+
# least one pipe after its first character; `\r?` as kramdown takes CRLF line ends
|
|
15
|
+
TABLE_SEPARATOR = /^[ \t]*\|?[ \t\-—–:]*\|[ \t\-—–:|]*\r?$/
|
|
16
|
+
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
# @param source [String] Markdown
|
|
20
|
+
# @return [String] the repaired Markdown
|
|
21
|
+
def call(source)
|
|
22
|
+
source.gsub(TABLE_SEPARATOR) { |line| line.tr('—–', '--') }
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'kramdown'
|
|
4
|
+
require 'kramdown-parser-gfm'
|
|
5
|
+
require_relative 'streaming_markdown_parser/version'
|
|
6
|
+
require_relative 'streaming_markdown_parser/listener'
|
|
7
|
+
require_relative 'streaming_markdown_parser/repair'
|
|
8
|
+
require_relative 'streaming_markdown_parser/closer'
|
|
9
|
+
|
|
10
|
+
# Parses Markdown as it streams in, an LLM reply delta by delta, into top-level blocks: each
|
|
11
|
+
# finished block reaches the listener once, final, and the unfinished tail is parsed again on
|
|
12
|
+
# every delta. Blocks are kramdown trees, so any kramdown converter renders them.
|
|
13
|
+
#
|
|
14
|
+
# class HtmlView
|
|
15
|
+
# include StreamingMarkdownParser::Listener
|
|
16
|
+
#
|
|
17
|
+
# def initialize
|
|
18
|
+
# @done = +''
|
|
19
|
+
# @tail = ''
|
|
20
|
+
# end
|
|
21
|
+
#
|
|
22
|
+
# def on_block(root) = @done << Kramdown::Converter::Html.convert(root).first
|
|
23
|
+
# def on_open(root) = @tail = Kramdown::Converter::Html.convert(root).first
|
|
24
|
+
# def to_html = @done + @tail
|
|
25
|
+
# end
|
|
26
|
+
#
|
|
27
|
+
# parser = StreamingMarkdownParser.new(view = HtmlView.new)
|
|
28
|
+
# parser.append("Hello.\n\nWor") # on_open: "Hello.", "Wor" (the next line may continue "Hello.")
|
|
29
|
+
# parser.append("ld.\n") # on_block: "Hello.", as "World." began a new block
|
|
30
|
+
# # on_open: "World."
|
|
31
|
+
# parser.finish # on_block: "World."; on_open: empty
|
|
32
|
+
# view.to_html # => "<p>Hello.</p>\n<p>World.</p>\n"
|
|
33
|
+
#
|
|
34
|
+
# A listener that wants only final output implements {Listener#on_block} alone.
|
|
35
|
+
#
|
|
36
|
+
# A reference link or footnote whose definition is a block away streams as literal text, and
|
|
37
|
+
# stays so: render the ended reply once more from {.parse}. See `D_references_stay_literal`.
|
|
38
|
+
#
|
|
39
|
+
# == Implementation details
|
|
40
|
+
#
|
|
41
|
+
# A cut between two blocks is taken only where kramdown agrees: once the first line after a
|
|
42
|
+
# blank line (or after a closer) is complete, the open text before it is parsed alone, the line
|
|
43
|
+
# alone, and both together, and the cut stands only if the blocks come out the same. So a loose
|
|
44
|
+
# list's next item, an indented continuation or a definition keeps the block open, with no
|
|
45
|
+
# Markdown rules written here.
|
|
46
|
+
#
|
|
47
|
+
# Inside an unclosed fence, `$$` math block, HTML comment or `{::comment}`-style extension there
|
|
48
|
+
# is no cut at all: kramdown reads paragraphs until the closer comes, and the closer would undo
|
|
49
|
+
# the cut. Whether a `$$`, `<!--` or extension line opens one, kramdown judges too, as a fence
|
|
50
|
+
# missed inside a block wrongly held open is a cut inside that fence.
|
|
51
|
+
#
|
|
52
|
+
# Per delta, the cost is one parse of the open tail, plus one check per candidate cut.
|
|
53
|
+
class StreamingMarkdownParser
|
|
54
|
+
# @return [Hash{Symbol => Object}] GFM input, as LLMs write backtick fences and pipe tables;
|
|
55
|
+
# reply paragraphs reflow, so no hard wraps
|
|
56
|
+
DEFAULT_OPTIONS = { input: 'GFM', hard_wrap: false }.freeze
|
|
57
|
+
|
|
58
|
+
# @return [Hash{Symbol => Object}] the kramdown options every parse runs with; pass them to a
|
|
59
|
+
# converter that needs them
|
|
60
|
+
attr_reader :options
|
|
61
|
+
|
|
62
|
+
# Parses a whole reply into one root: one that arrived whole, or a streamed one once it ends,
|
|
63
|
+
# as only the whole text resolves a reference whose definition is a block away.
|
|
64
|
+
#
|
|
65
|
+
# root = StreamingMarkdownParser.parse("Hello.\n\nWorld.\n")
|
|
66
|
+
# Kramdown::Converter::Html.convert(root).first # => "<p>Hello.</p>\n<p>World.</p>\n"
|
|
67
|
+
#
|
|
68
|
+
# A listener takes a whole reply through {#finish} instead: `new(listener).finish(text)`.
|
|
69
|
+
# kramdown's warnings are dropped, as a stream's parses of fragments can't report the same ones.
|
|
70
|
+
#
|
|
71
|
+
# @param text [String]
|
|
72
|
+
# @param options [Hash{Symbol => Object}] kramdown options, merged over {DEFAULT_OPTIONS}
|
|
73
|
+
# @return [Kramdown::Element] the root, {Repair} applied and blank-line elements dropped
|
|
74
|
+
def self.parse(text, **options)
|
|
75
|
+
root = Kramdown::Document.new(Repair.call(text), **DEFAULT_OPTIONS, **options).root
|
|
76
|
+
root.children.reject! { |child| child.type == :blank }
|
|
77
|
+
root
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# @param listener [StreamingMarkdownParser::Listener] receives the blocks
|
|
81
|
+
# @param options [Hash{Symbol => Object}] kramdown options, merged over {DEFAULT_OPTIONS}
|
|
82
|
+
def initialize(listener, **options)
|
|
83
|
+
@listener = listener
|
|
84
|
+
@options = DEFAULT_OPTIONS.merge(options).freeze
|
|
85
|
+
@source = +''
|
|
86
|
+
@open_from = 0 # where the open tail starts in @source
|
|
87
|
+
@scan = 0 # the first line of @source not yet scanned
|
|
88
|
+
@closer = nil # the open block's Closer, nil outside one
|
|
89
|
+
@boundary = false # the last scanned line ended a block, if kramdown agrees
|
|
90
|
+
@finished = false
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Feeds one delta: commits every block it finishes, then re-parses the open tail. A nil or
|
|
94
|
+
# empty delta does nothing.
|
|
95
|
+
#
|
|
96
|
+
# @param delta [String, nil]
|
|
97
|
+
# @return [void]
|
|
98
|
+
# @raise [RuntimeError] after {#finish}
|
|
99
|
+
def append(delta)
|
|
100
|
+
raise 'StreamingMarkdownParser#append after #finish' if @finished
|
|
101
|
+
return if delta.nil? || delta.empty?
|
|
102
|
+
|
|
103
|
+
@source << delta
|
|
104
|
+
notify(scan, parse(@source[@open_from..]))
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Ends the stream: the open tail becomes blocks, and an empty {Listener#on_open} clears it.
|
|
108
|
+
#
|
|
109
|
+
# The streamed deltas may not add up to the final text (pikuri's deltas are a preview of its
|
|
110
|
+
# final message). Given a different `authoritative`, the listener gets {Listener#on_reset} and
|
|
111
|
+
# then every block of the authoritative text. With no deltas before, there is nothing to reset:
|
|
112
|
+
#
|
|
113
|
+
# StreamingMarkdownParser.new(listener).finish(text) # a whole reply: its blocks, no reset
|
|
114
|
+
#
|
|
115
|
+
# Given the streamed text, it parses nothing again, so its references stay as they streamed.
|
|
116
|
+
#
|
|
117
|
+
# @param authoritative [String, nil] the final text, nil for the deltas as they came
|
|
118
|
+
# @return [void]
|
|
119
|
+
# @raise [RuntimeError] if called twice
|
|
120
|
+
def finish(authoritative = nil)
|
|
121
|
+
raise 'StreamingMarkdownParser#finish called twice' if @finished
|
|
122
|
+
|
|
123
|
+
@finished = true
|
|
124
|
+
reset = false
|
|
125
|
+
if !authoritative.nil? && authoritative != @source
|
|
126
|
+
reset = !@source.empty?
|
|
127
|
+
@source = authoritative.dup
|
|
128
|
+
@open_from = 0
|
|
129
|
+
end
|
|
130
|
+
notify(commit(@source.size), parse(''), reset:)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# @return [Boolean] true after {#finish}
|
|
134
|
+
def finished? = @finished
|
|
135
|
+
|
|
136
|
+
# @return [String] the text so far: every delta, or the authoritative text after {#finish}
|
|
137
|
+
def source = @source.dup
|
|
138
|
+
|
|
139
|
+
private
|
|
140
|
+
|
|
141
|
+
# Walks every complete line not yet scanned, committing at each cut kramdown agrees to.
|
|
142
|
+
#
|
|
143
|
+
# @return [Array<Kramdown::Element>] the committed blocks, for {#notify}
|
|
144
|
+
def scan
|
|
145
|
+
blocks = []
|
|
146
|
+
while (newline = @source.index("\n", @scan))
|
|
147
|
+
start = @scan
|
|
148
|
+
@scan = newline + 1
|
|
149
|
+
line = @source[start...@scan]
|
|
150
|
+
next if @closer && watch(line)
|
|
151
|
+
|
|
152
|
+
if line.strip.empty?
|
|
153
|
+
@boundary = true
|
|
154
|
+
else
|
|
155
|
+
blocks.concat(commit(start)) if @boundary && start > @open_from && clean_cut?(start, @scan)
|
|
156
|
+
@boundary = false
|
|
157
|
+
open_block(line, start)
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
blocks
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# @param cut [Integer] a line start in @source
|
|
164
|
+
# @param line_end [Integer] the end of the line that starts at `cut`
|
|
165
|
+
# @return [Boolean] true if kramdown parses the open text before `cut` and the line after it
|
|
166
|
+
# alone into the same blocks as together
|
|
167
|
+
def clean_cut?(cut, line_end)
|
|
168
|
+
head = @source[@open_from...cut]
|
|
169
|
+
line = @source[cut...line_end]
|
|
170
|
+
shape(parse(head + line)) == shape(parse(head)) + shape(parse(line))
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Ends the open text at `cut`: the open tail starts there.
|
|
174
|
+
#
|
|
175
|
+
# @param cut [Integer]
|
|
176
|
+
# @return [Array<Kramdown::Element>] every block of the text before `cut`, each in its own root
|
|
177
|
+
def commit(cut)
|
|
178
|
+
root = parse(@source[@open_from...cut])
|
|
179
|
+
@open_from = cut
|
|
180
|
+
root.children.map do |block|
|
|
181
|
+
single = Kramdown::Element.new(:root, nil, nil, root.options.dup)
|
|
182
|
+
single.children << block
|
|
183
|
+
single
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Calls the listener, the last step of {#append} and {#finish}: the parser's state is already
|
|
188
|
+
# final, so a listener that raises leaves nothing half-updated, and no block comes twice.
|
|
189
|
+
#
|
|
190
|
+
# @param blocks [Array<Kramdown::Element>] for {Listener#on_block}
|
|
191
|
+
# @param open [Kramdown::Element] for {Listener#on_open}
|
|
192
|
+
# @param reset [Boolean] true to call {Listener#on_reset} first
|
|
193
|
+
# @return [void]
|
|
194
|
+
def notify(blocks, open, reset: false)
|
|
195
|
+
@listener.on_reset if reset
|
|
196
|
+
blocks.each { |block| @listener.on_block(block) }
|
|
197
|
+
@listener.on_open(open)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# @param text [String]
|
|
201
|
+
# @return [Kramdown::Element] see {.parse}
|
|
202
|
+
def parse(text) = self.class.parse(text, **@options)
|
|
203
|
+
|
|
204
|
+
# @param root [Kramdown::Element]
|
|
205
|
+
# @return [Array<Array>] each block as nested arrays, line numbers left out, for comparing
|
|
206
|
+
def shape(root) = root.children.map { |child| element_shape(child) }
|
|
207
|
+
|
|
208
|
+
# @param element [Kramdown::Element]
|
|
209
|
+
# @return [Array] a value that is itself an element, a footnote's definition, as its shape too:
|
|
210
|
+
# elements compare by identity, so two parses of one footnote would never agree on a cut
|
|
211
|
+
def element_shape(element)
|
|
212
|
+
value = element.value.is_a?(Kramdown::Element) ? element_shape(element.value) : element.value
|
|
213
|
+
[element.type, value, element.attr, element.options.except(:location),
|
|
214
|
+
element.children.map { |child| element_shape(child) }]
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# @param start [Integer] where an opener line starts in @source
|
|
218
|
+
# @param closed [String] that opener, closed on the spot
|
|
219
|
+
# @return [Boolean] true if kramdown reads an opener at `start`: the open text before it, then
|
|
220
|
+
# `closed`, parse into the blocks of each alone. After a paragraph line, `$$` is inline math.
|
|
221
|
+
def opens?(start, closed)
|
|
222
|
+
head = @source[@open_from...start]
|
|
223
|
+
types(parse(head + closed)) == types(parse(head)) + types(parse(closed))
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# @param root [Kramdown::Element]
|
|
227
|
+
# @return [Array<Symbol>] the type of each block
|
|
228
|
+
def types(root) = root.children.map(&:type)
|
|
229
|
+
|
|
230
|
+
# Sets {@closer} if `line` opens a block that only its closer ends, and kramdown agrees.
|
|
231
|
+
#
|
|
232
|
+
# @param line [String]
|
|
233
|
+
# @param start [Integer] where `line` starts in @source
|
|
234
|
+
# @return [void]
|
|
235
|
+
def open_block(line, start)
|
|
236
|
+
closer, rest = Closer.open(line, @scan)
|
|
237
|
+
return unless closer
|
|
238
|
+
return if closer.closed_opener && !opens?(start, closer.closed_opener)
|
|
239
|
+
|
|
240
|
+
@closer = closer
|
|
241
|
+
watch(rest)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Hands `line` to the open block's closer and takes what it answers; see {Closer}.
|
|
245
|
+
#
|
|
246
|
+
# @param line [String]
|
|
247
|
+
# @return [Boolean] false if `line` comes after the block, to scan like any other
|
|
248
|
+
def watch(line)
|
|
249
|
+
after = @closer.watch(line)
|
|
250
|
+
case after
|
|
251
|
+
when Closer::PASS
|
|
252
|
+
@closer = nil
|
|
253
|
+
return false
|
|
254
|
+
when Closer::Rewind
|
|
255
|
+
@scan = after.to
|
|
256
|
+
@closer = nil
|
|
257
|
+
@boundary = false
|
|
258
|
+
else
|
|
259
|
+
@boundary = true unless after.equal?(@closer)
|
|
260
|
+
@closer = after
|
|
261
|
+
end
|
|
262
|
+
true
|
|
263
|
+
end
|
|
264
|
+
end
|
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
# Parses Markdown as it streams in, an LLM reply delta by delta, into top-level blocks: each
|
|
2
|
+
# finished block reaches the listener once, final, and the unfinished tail is parsed again on
|
|
3
|
+
# every delta. Blocks are kramdown trees, so any kramdown converter renders them.
|
|
4
|
+
#
|
|
5
|
+
# class HtmlView
|
|
6
|
+
# include StreamingMarkdownParser::Listener
|
|
7
|
+
#
|
|
8
|
+
# def initialize
|
|
9
|
+
# @done = +''
|
|
10
|
+
# @tail = ''
|
|
11
|
+
# end
|
|
12
|
+
#
|
|
13
|
+
# def on_block(root) = @done << Kramdown::Converter::Html.convert(root).first
|
|
14
|
+
# def on_open(root) = @tail = Kramdown::Converter::Html.convert(root).first
|
|
15
|
+
# def to_html = @done + @tail
|
|
16
|
+
# end
|
|
17
|
+
#
|
|
18
|
+
# parser = StreamingMarkdownParser.new(view = HtmlView.new)
|
|
19
|
+
# parser.append("Hello.\n\nWor") # on_open: "Hello.", "Wor" (the next line may continue "Hello.")
|
|
20
|
+
# parser.append("ld.\n") # on_block: "Hello.", as "World." began a new block
|
|
21
|
+
# # on_open: "World."
|
|
22
|
+
# parser.finish # on_block: "World."; on_open: empty
|
|
23
|
+
# view.to_html # => "<p>Hello.</p>\n<p>World.</p>\n"
|
|
24
|
+
#
|
|
25
|
+
# A listener that wants only final output implements {Listener#on_block} alone.
|
|
26
|
+
#
|
|
27
|
+
# A reference link or footnote whose definition is a block away streams as literal text, and
|
|
28
|
+
# stays so: render the ended reply once more from {.parse}. See `D_references_stay_literal`.
|
|
29
|
+
#
|
|
30
|
+
# == Implementation details
|
|
31
|
+
#
|
|
32
|
+
# A cut between two blocks is taken only where kramdown agrees: once the first line after a
|
|
33
|
+
# blank line (or after a closer) is complete, the open text before it is parsed alone, the line
|
|
34
|
+
# alone, and both together, and the cut stands only if the blocks come out the same. So a loose
|
|
35
|
+
# list's next item, an indented continuation or a definition keeps the block open, with no
|
|
36
|
+
# Markdown rules written here.
|
|
37
|
+
#
|
|
38
|
+
# Inside an unclosed fence, `$$` math block, HTML comment or `{::comment}`-style extension there
|
|
39
|
+
# is no cut at all: kramdown reads paragraphs until the closer comes, and the closer would undo
|
|
40
|
+
# the cut. Whether a `$$`, `<!--` or extension line opens one, kramdown judges too, as a fence
|
|
41
|
+
# missed inside a block wrongly held open is a cut inside that fence.
|
|
42
|
+
#
|
|
43
|
+
# Per delta, the cost is one parse of the open tail, plus one check per candidate cut.
|
|
44
|
+
class StreamingMarkdownParser
|
|
45
|
+
DEFAULT_OPTIONS: ::Hash[Symbol, Object]
|
|
46
|
+
VERSION: String
|
|
47
|
+
|
|
48
|
+
# Parses a whole reply into one root: one that arrived whole, or a streamed one once it ends,
|
|
49
|
+
# as only the whole text resolves a reference whose definition is a block away.
|
|
50
|
+
#
|
|
51
|
+
# root = StreamingMarkdownParser.parse("Hello.\n\nWorld.\n")
|
|
52
|
+
# Kramdown::Converter::Html.convert(root).first # => "<p>Hello.</p>\n<p>World.</p>\n"
|
|
53
|
+
#
|
|
54
|
+
# A listener takes a whole reply through {#finish} instead: `new(listener).finish(text)`.
|
|
55
|
+
# kramdown's warnings are dropped, as a stream's parses of fragments can't report the same ones.
|
|
56
|
+
#
|
|
57
|
+
# _@param_ `text`
|
|
58
|
+
#
|
|
59
|
+
# _@param_ `options` — kramdown options, merged over {DEFAULT_OPTIONS}
|
|
60
|
+
#
|
|
61
|
+
# _@return_ — the root, {Repair} applied and blank-line elements dropped
|
|
62
|
+
def self.parse: (String text, **::Hash[Symbol, Object] options) -> Kramdown::Element
|
|
63
|
+
|
|
64
|
+
# _@param_ `listener` — receives the blocks
|
|
65
|
+
#
|
|
66
|
+
# _@param_ `options` — kramdown options, merged over {DEFAULT_OPTIONS}
|
|
67
|
+
def initialize: (StreamingMarkdownParser::Listener listener, **::Hash[Symbol, Object] options) -> void
|
|
68
|
+
|
|
69
|
+
# Feeds one delta: commits every block it finishes, then re-parses the open tail. A nil or
|
|
70
|
+
# empty delta does nothing.
|
|
71
|
+
#
|
|
72
|
+
# _@param_ `delta`
|
|
73
|
+
def append: (String? delta) -> void
|
|
74
|
+
|
|
75
|
+
# Ends the stream: the open tail becomes blocks, and an empty {Listener#on_open} clears it.
|
|
76
|
+
#
|
|
77
|
+
# The streamed deltas may not add up to the final text (pikuri's deltas are a preview of its
|
|
78
|
+
# final message). Given a different `authoritative`, the listener gets {Listener#on_reset} and
|
|
79
|
+
# then every block of the authoritative text. With no deltas before, there is nothing to reset:
|
|
80
|
+
#
|
|
81
|
+
# StreamingMarkdownParser.new(listener).finish(text) # a whole reply: its blocks, no reset
|
|
82
|
+
#
|
|
83
|
+
# Given the streamed text, it parses nothing again, so its references stay as they streamed.
|
|
84
|
+
#
|
|
85
|
+
# _@param_ `authoritative` — the final text, nil for the deltas as they came
|
|
86
|
+
def finish: (?String? authoritative) -> void
|
|
87
|
+
|
|
88
|
+
# _@return_ — true after {#finish}
|
|
89
|
+
def finished?: () -> bool
|
|
90
|
+
|
|
91
|
+
# _@return_ — the text so far: every delta, or the authoritative text after {#finish}
|
|
92
|
+
def source: () -> String
|
|
93
|
+
|
|
94
|
+
# Walks every complete line not yet scanned, committing at each cut kramdown agrees to.
|
|
95
|
+
#
|
|
96
|
+
# _@return_ — the committed blocks, for {#notify}
|
|
97
|
+
def scan: () -> ::Array[Kramdown::Element]
|
|
98
|
+
|
|
99
|
+
# _@param_ `cut` — a line start in @source
|
|
100
|
+
#
|
|
101
|
+
# _@param_ `line_end` — the end of the line that starts at `cut`
|
|
102
|
+
#
|
|
103
|
+
# _@return_ — true if kramdown parses the open text before `cut` and the line after it
|
|
104
|
+
# alone into the same blocks as together
|
|
105
|
+
def clean_cut?: (Integer cut, Integer line_end) -> bool
|
|
106
|
+
|
|
107
|
+
# Ends the open text at `cut`: the open tail starts there.
|
|
108
|
+
#
|
|
109
|
+
# _@param_ `cut`
|
|
110
|
+
#
|
|
111
|
+
# _@return_ — every block of the text before `cut`, each in its own root
|
|
112
|
+
def commit: (Integer cut) -> ::Array[Kramdown::Element]
|
|
113
|
+
|
|
114
|
+
# Calls the listener, the last step of {#append} and {#finish}: the parser's state is already
|
|
115
|
+
# final, so a listener that raises leaves nothing half-updated, and no block comes twice.
|
|
116
|
+
#
|
|
117
|
+
# _@param_ `blocks` — for {Listener#on_block}
|
|
118
|
+
#
|
|
119
|
+
# _@param_ `open` — for {Listener#on_open}
|
|
120
|
+
#
|
|
121
|
+
# _@param_ `reset` — true to call {Listener#on_reset} first
|
|
122
|
+
def notify: (::Array[Kramdown::Element] blocks, Kramdown::Element open, ?reset: bool) -> void
|
|
123
|
+
|
|
124
|
+
# _@param_ `text`
|
|
125
|
+
#
|
|
126
|
+
# _@return_ — see {.parse}
|
|
127
|
+
def parse: (String text) -> Kramdown::Element
|
|
128
|
+
|
|
129
|
+
# _@param_ `root`
|
|
130
|
+
#
|
|
131
|
+
# _@return_ — each block as nested arrays, line numbers left out, for comparing
|
|
132
|
+
def shape: (Kramdown::Element root) -> ::Array[::Array[untyped]]
|
|
133
|
+
|
|
134
|
+
# _@param_ `element`
|
|
135
|
+
#
|
|
136
|
+
# _@return_ — a value that is itself an element, a footnote's definition, as its shape too:
|
|
137
|
+
# elements compare by identity, so two parses of one footnote would never agree on a cut
|
|
138
|
+
def element_shape: (Kramdown::Element element) -> ::Array[untyped]
|
|
139
|
+
|
|
140
|
+
# _@param_ `start` — where an opener line starts in @source
|
|
141
|
+
#
|
|
142
|
+
# _@param_ `closed` — that opener, closed on the spot
|
|
143
|
+
#
|
|
144
|
+
# _@return_ — true if kramdown reads an opener at `start`: the open text before it, then
|
|
145
|
+
# `closed`, parse into the blocks of each alone. After a paragraph line, `$$` is inline math.
|
|
146
|
+
def opens?: (Integer start, String closed) -> bool
|
|
147
|
+
|
|
148
|
+
# _@param_ `root`
|
|
149
|
+
#
|
|
150
|
+
# _@return_ — the type of each block
|
|
151
|
+
def types: (Kramdown::Element root) -> ::Array[Symbol]
|
|
152
|
+
|
|
153
|
+
# Sets {@closer} if `line` opens a block that only its closer ends, and kramdown agrees.
|
|
154
|
+
#
|
|
155
|
+
# _@param_ `line`
|
|
156
|
+
#
|
|
157
|
+
# _@param_ `start` — where `line` starts in @source
|
|
158
|
+
def open_block: (String line, Integer start) -> void
|
|
159
|
+
|
|
160
|
+
# Hands `line` to the open block's closer and takes what it answers; see {Closer}.
|
|
161
|
+
#
|
|
162
|
+
# _@param_ `line`
|
|
163
|
+
#
|
|
164
|
+
# _@return_ — false if `line` comes after the block, to scan like any other
|
|
165
|
+
def watch: (String line) -> bool
|
|
166
|
+
|
|
167
|
+
# _@return_ — the kramdown options every parse runs with; pass them to a
|
|
168
|
+
# converter that needs them
|
|
169
|
+
attr_reader options: ::Hash[Symbol, Object]
|
|
170
|
+
|
|
171
|
+
# The blocks that only their closer ends: fences, `$$` math, HTML comments and kramdown's
|
|
172
|
+
# extensions. Until the closer comes, kramdown reads paragraphs, so the parser takes no cut
|
|
173
|
+
# inside one. It holds the open block's closer and hands it each line, getting back the closer
|
|
174
|
+
# for the next:
|
|
175
|
+
#
|
|
176
|
+
# closer, rest = Closer.open("$$ a = b\n", 9) # a Closer::Math; rest is " a = b\n"
|
|
177
|
+
# closer = closer.watch(rest) # itself: still open
|
|
178
|
+
# closer = closer.watch("c = d $$\n") # a Closer::MathKept: math if the next line agrees
|
|
179
|
+
# closer.watch("next\n") # Closer::Rewind[to: 9]: kramdown reverts it
|
|
180
|
+
#
|
|
181
|
+
# `watch` answers itself while the block is open, nil once it closed on that line, {PASS} when
|
|
182
|
+
# the line comes after the block and is scanned like any other, or a {Rewind} when kramdown
|
|
183
|
+
# reverts the block to paragraphs, so a fence inside it counts after all.
|
|
184
|
+
#
|
|
185
|
+
# Each closer is kramdown's own; one too loose takes a cut that the real closer then undoes.
|
|
186
|
+
module Closer
|
|
187
|
+
PASS: Symbol
|
|
188
|
+
EXTENSION: Regexp
|
|
189
|
+
MATH_KEPT: Regexp
|
|
190
|
+
|
|
191
|
+
# _@param_ `line` — a line where a block may start
|
|
192
|
+
#
|
|
193
|
+
# _@param_ `body` — where the line after it starts in the source
|
|
194
|
+
#
|
|
195
|
+
# _@return_ — the closer `line` opens, if any, and the rest of `line`
|
|
196
|
+
# after the opener, for that closer's first {#watch}
|
|
197
|
+
def self.open: (String line, Integer body) -> [Object, String]?
|
|
198
|
+
|
|
199
|
+
# Ends at a whole line: a fence of the same character, at least as long, or an extension's
|
|
200
|
+
# `{:/name}` or `{:/}`.
|
|
201
|
+
#
|
|
202
|
+
# @!attribute [r] pattern
|
|
203
|
+
# @return [Regexp] the closing line
|
|
204
|
+
# @!attribute [r] closed_opener
|
|
205
|
+
# @return [String, nil] the opener closed on the spot, for kramdown to confirm it opens a
|
|
206
|
+
# block; nil when no check is needed
|
|
207
|
+
class Line < Data
|
|
208
|
+
# _@param_ `line`
|
|
209
|
+
#
|
|
210
|
+
# _@return_ — see {Closer}
|
|
211
|
+
def watch: (String line) -> Object?
|
|
212
|
+
|
|
213
|
+
# _@return_ — the closing line
|
|
214
|
+
attr_reader pattern: Regexp
|
|
215
|
+
|
|
216
|
+
# _@return_ — the opener closed on the spot, for kramdown to confirm it opens a
|
|
217
|
+
# block; nil when no check is needed
|
|
218
|
+
attr_reader closed_opener: String?
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Ends at the first `-->`, mid-line too. kramdown checks nothing after it.
|
|
222
|
+
#
|
|
223
|
+
# @!attribute [r] opener
|
|
224
|
+
# @return [String] `<!--` and the spaces before it
|
|
225
|
+
class Comment < Data
|
|
226
|
+
# _@return_ — see {Line#closed_opener}
|
|
227
|
+
def closed_opener: () -> String
|
|
228
|
+
|
|
229
|
+
# _@param_ `line`
|
|
230
|
+
#
|
|
231
|
+
# _@return_ — see {Closer}
|
|
232
|
+
def watch: (String line) -> Object?
|
|
233
|
+
|
|
234
|
+
# _@return_ — `<!--` and the spaces before it
|
|
235
|
+
attr_reader opener: String
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# Ends at the first `$$`, mid-line too; text after it on its line reverts the block.
|
|
239
|
+
#
|
|
240
|
+
# @!attribute [r] opener
|
|
241
|
+
# @return [String] `$$` and the spaces before it
|
|
242
|
+
# @!attribute [r] body
|
|
243
|
+
# @return [Integer] where the line after the opener starts, to rescan from
|
|
244
|
+
class Math < Data
|
|
245
|
+
# _@return_ — see {Line#closed_opener}
|
|
246
|
+
def closed_opener: () -> String
|
|
247
|
+
|
|
248
|
+
# _@param_ `line`
|
|
249
|
+
#
|
|
250
|
+
# _@return_ — see {Closer}
|
|
251
|
+
def watch: (String line) -> Object?
|
|
252
|
+
|
|
253
|
+
# _@return_ — `$$` and the spaces before it
|
|
254
|
+
attr_reader opener: String
|
|
255
|
+
|
|
256
|
+
# _@return_ — where the line after the opener starts, to rescan from
|
|
257
|
+
attr_reader body: Integer
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# Judges the line after a `$$` block's closer: math only if it is blank, an IAL or `^`.
|
|
261
|
+
#
|
|
262
|
+
# @!attribute [r] body
|
|
263
|
+
# @return [Integer] see {Math#body}
|
|
264
|
+
class MathKept < Data
|
|
265
|
+
# _@param_ `line`
|
|
266
|
+
#
|
|
267
|
+
# _@return_ — see {Closer}
|
|
268
|
+
def watch: (String line) -> Object
|
|
269
|
+
|
|
270
|
+
# _@return_ — see {Math#body}
|
|
271
|
+
attr_reader body: Integer
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
# kramdown reverted the block: scan again from `to`, with no block open.
|
|
275
|
+
#
|
|
276
|
+
# @!attribute [r] to
|
|
277
|
+
# @return [Integer] where in the source to scan from
|
|
278
|
+
class Rewind < Data
|
|
279
|
+
# _@return_ — where in the source to scan from
|
|
280
|
+
attr_reader to: Integer
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# Fixes what LLMs get wrong in Markdown before kramdown parses it. Today one thing: a table
|
|
285
|
+
# separator row written with typographic dashes, which kramdown won't take, so the table falls
|
|
286
|
+
# apart into paragraphs of literal pipes.
|
|
287
|
+
#
|
|
288
|
+
# Repair.call("| a | b |\n|—-|—-|\n| 1 | 2 |") # => "| a | b |\n|--|--|\n| 1 | 2 |"
|
|
289
|
+
#
|
|
290
|
+
# Only a line made wholly of separator characters with an inner pipe is touched, so prose with
|
|
291
|
+
# an em-dash and a `---` rule stay as they are.
|
|
292
|
+
module Repair
|
|
293
|
+
TABLE_SEPARATOR: Regexp
|
|
294
|
+
|
|
295
|
+
# _@param_ `source` — Markdown
|
|
296
|
+
#
|
|
297
|
+
# _@return_ — the repaired Markdown
|
|
298
|
+
def call: (String source) -> String
|
|
299
|
+
|
|
300
|
+
# _@param_ `source` — Markdown
|
|
301
|
+
#
|
|
302
|
+
# _@return_ — the repaired Markdown
|
|
303
|
+
def self.call: (String source) -> String
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# What a {StreamingMarkdownParser} calls. Include it and override what you use; the rest do
|
|
307
|
+
# nothing. Each call gets a kramdown root, which a converter takes:
|
|
308
|
+
#
|
|
309
|
+
# def on_block(root) = @html << Kramdown::Converter::Html.convert(root).first
|
|
310
|
+
#
|
|
311
|
+
# Calls come on the thread that called {StreamingMarkdownParser#append} or
|
|
312
|
+
# {StreamingMarkdownParser#finish}.
|
|
313
|
+
#
|
|
314
|
+
# Don't raise. A raise propagates out of that `append` or `finish`, and the calls it still owed
|
|
315
|
+
# are lost, not sent again: the parser goes on, but the listener's copy of the reply has a gap.
|
|
316
|
+
module Listener
|
|
317
|
+
# One finished top-level block (a paragraph, a heading, a whole list), a root with that block
|
|
318
|
+
# as its only child. Final: it is never called back or changed, except by {#on_reset}.
|
|
319
|
+
#
|
|
320
|
+
# _@param_ `root`
|
|
321
|
+
def on_block: (Kramdown::Element root) -> void
|
|
322
|
+
|
|
323
|
+
# The unfinished tail after the last finished block, parsed as it stands: zero or more
|
|
324
|
+
# blocks, provisional. Each call replaces the one before; after the stream ends, it comes
|
|
325
|
+
# empty.
|
|
326
|
+
#
|
|
327
|
+
# _@param_ `root`
|
|
328
|
+
def on_open: (Kramdown::Element root) -> void
|
|
329
|
+
|
|
330
|
+
# The final text differed from the streamed one: forget every block so far, all of them come
|
|
331
|
+
# again.
|
|
332
|
+
def on_reset: () -> void
|
|
333
|
+
end
|
|
334
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: streaming-markdown-parser
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Martin Vysny
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: kramdown
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '2.5'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '2.5'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: kramdown-parser-gfm
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - "~>"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '1.1'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '1.1'
|
|
40
|
+
executables: []
|
|
41
|
+
extensions: []
|
|
42
|
+
extra_rdoc_files: []
|
|
43
|
+
files:
|
|
44
|
+
- LICENSE
|
|
45
|
+
- README.md
|
|
46
|
+
- lib/streaming_markdown_parser.rb
|
|
47
|
+
- lib/streaming_markdown_parser/closer.rb
|
|
48
|
+
- lib/streaming_markdown_parser/listener.rb
|
|
49
|
+
- lib/streaming_markdown_parser/repair.rb
|
|
50
|
+
- lib/streaming_markdown_parser/version.rb
|
|
51
|
+
- sig/streaming_markdown_parser.rbs
|
|
52
|
+
homepage: https://github.com/mvysny/streaming-markdown-parser
|
|
53
|
+
licenses:
|
|
54
|
+
- MIT
|
|
55
|
+
metadata:
|
|
56
|
+
rubygems_mfa_required: 'true'
|
|
57
|
+
rdoc_options: []
|
|
58
|
+
require_paths:
|
|
59
|
+
- lib
|
|
60
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
61
|
+
requirements:
|
|
62
|
+
- - ">="
|
|
63
|
+
- !ruby/object:Gem::Version
|
|
64
|
+
version: '3.3'
|
|
65
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
66
|
+
requirements:
|
|
67
|
+
- - ">="
|
|
68
|
+
- !ruby/object:Gem::Version
|
|
69
|
+
version: '0'
|
|
70
|
+
requirements: []
|
|
71
|
+
rubygems_version: 3.6.7
|
|
72
|
+
specification_version: 4
|
|
73
|
+
summary: Parses streamed Markdown, an LLM reply delta by delta, into final kramdown
|
|
74
|
+
blocks
|
|
75
|
+
test_files: []
|