token_reel 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/ci.yml +35 -0
- data/.gitignore +5 -0
- data/Gemfile +5 -0
- data/README.md +170 -0
- data/Rakefile +59 -0
- data/exe/token_reel +7 -0
- data/lib/token_reel/cli.rb +118 -0
- data/lib/token_reel/config.rb +56 -0
- data/lib/token_reel/errors.rb +7 -0
- data/lib/token_reel/fonts.rb +48 -0
- data/lib/token_reel/generator.rb +71 -0
- data/lib/token_reel/gif_writer.rb +23 -0
- data/lib/token_reel/highlight.rb +62 -0
- data/lib/token_reel/renderer.rb +273 -0
- data/lib/token_reel/script.rb +86 -0
- data/lib/token_reel/theme.rb +38 -0
- data/lib/token_reel/timeline.rb +120 -0
- data/lib/token_reel/tokenizer.rb +24 -0
- data/lib/token_reel/version.rb +5 -0
- data/lib/token_reel.rb +31 -0
- data/token_reel.gemspec +40 -0
- metadata +102 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TokenReel
|
|
4
|
+
# A small, dependency-free syntax colorizer for text inside fenced
|
|
5
|
+
# code blocks. It isn't a real per-language grammar -- just enough
|
|
6
|
+
# regex-based token classing (comments, strings, numbers, a shared
|
|
7
|
+
# keyword list spanning a handful of mainstream languages) to make
|
|
8
|
+
# demo code read as code instead of one flat color. The language tag
|
|
9
|
+
# on a fence (```ruby) is cosmetic; every fence is highlighted the
|
|
10
|
+
# same way.
|
|
11
|
+
module Highlight
|
|
12
|
+
KEYWORDS = %w[
|
|
13
|
+
def end class module function fn func return if elif elsif else
|
|
14
|
+
unless while until for do begin rescue ensure raise throw try
|
|
15
|
+
catch except finally break next continue yield case when switch
|
|
16
|
+
match default import export from require include use pub package
|
|
17
|
+
namespace let const var static final public private protected
|
|
18
|
+
abstract new self this super nil null none true false async await
|
|
19
|
+
int float double bool bool8 string char void byte long short
|
|
20
|
+
struct enum interface extends implements typeof instanceof
|
|
21
|
+
in of as with lambda del pass global nonlocal
|
|
22
|
+
].freeze
|
|
23
|
+
|
|
24
|
+
# One token per iteration: a comment to end-of-line, a quoted
|
|
25
|
+
# string (single/double/backtick, backslash-escaped), an integer
|
|
26
|
+
# or float, an identifier/keyword, or a single other character
|
|
27
|
+
# (whitespace, punctuation, operators).
|
|
28
|
+
TOKEN = /
|
|
29
|
+
(?<comment>\#.*\z|\/\/.*\z)
|
|
30
|
+
|(?<string>"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`(?:[^`\\]|\\.)*`)
|
|
31
|
+
|(?<number>\b\d+(?:\.\d+)?\b)
|
|
32
|
+
|(?<word>[A-Za-z_][A-Za-z0-9_]*)
|
|
33
|
+
|(?<other>.)
|
|
34
|
+
/x
|
|
35
|
+
|
|
36
|
+
# Splits one line of code into color-tagged spans. Adjacent tokens
|
|
37
|
+
# that land on the same color are merged, so a run of plain text
|
|
38
|
+
# becomes one span instead of one per character.
|
|
39
|
+
def self.spans(line, palette)
|
|
40
|
+
out = []
|
|
41
|
+
line.scan(TOKEN) do
|
|
42
|
+
m = Regexp.last_match
|
|
43
|
+
text, color = classify(m, palette)
|
|
44
|
+
if out.any? && out.last[:color] == color
|
|
45
|
+
out.last[:text] += text
|
|
46
|
+
else
|
|
47
|
+
out << { text: text, color: color }
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
out
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def self.classify(m, palette)
|
|
54
|
+
return [m[:comment], palette[:syn_comment]] if m[:comment]
|
|
55
|
+
return [m[:string], palette[:syn_string]] if m[:string]
|
|
56
|
+
return [m[:number], palette[:syn_number]] if m[:number]
|
|
57
|
+
return [m[:word], KEYWORDS.include?(m[:word]) ? palette[:syn_keyword] : palette[:fg]] if m[:word]
|
|
58
|
+
|
|
59
|
+
[m[:other], palette[:fg]]
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "open3"
|
|
4
|
+
|
|
5
|
+
module TokenReel
|
|
6
|
+
# Turns a Timeline::State into a terminal-window PNG. Text layout is
|
|
7
|
+
# done in Ruby (word wrap, cursor placement); ImageMagick just draws
|
|
8
|
+
# the rectangles and glyphs we tell it to.
|
|
9
|
+
class Renderer
|
|
10
|
+
HEADER_H = 40
|
|
11
|
+
PAD_X = 18
|
|
12
|
+
PAD_Y = 14
|
|
13
|
+
DOT_R = 6
|
|
14
|
+
|
|
15
|
+
attr_reader :char_w, :char_h, :total_lines
|
|
16
|
+
|
|
17
|
+
def self.convert_binary
|
|
18
|
+
@convert_binary ||= %w[magick convert].find { |bin| system("which #{bin} > /dev/null 2>&1") } ||
|
|
19
|
+
raise(RenderError, "ImageMagick not found: install `imagemagick` (needs `convert` or `magick` on PATH)")
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def initialize(config, timeline)
|
|
23
|
+
@config = config
|
|
24
|
+
@timeline = timeline
|
|
25
|
+
@palette = Theme.fetch(config.theme)
|
|
26
|
+
@font = Fonts.resolve!(config.font)
|
|
27
|
+
calibrate!
|
|
28
|
+
# The window is a fixed config.rows lines tall, like a real
|
|
29
|
+
# console -- every frame shares this height regardless of how
|
|
30
|
+
# much text it holds.
|
|
31
|
+
@total_lines = @config.rows
|
|
32
|
+
@width = @config.cols * char_w + PAD_X * 2
|
|
33
|
+
@height = HEADER_H + PAD_Y * 2 + total_lines * char_h
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def render(state, out_path)
|
|
37
|
+
lines = wrap_body(state)
|
|
38
|
+
# Once a frame's content outgrows the window, scroll: keep only
|
|
39
|
+
# the most recent total_lines lines, same as a real terminal
|
|
40
|
+
# dropping its oldest lines off the top.
|
|
41
|
+
lines = lines.last(total_lines) if lines.size > total_lines
|
|
42
|
+
pad = [total_lines - lines.size, 0].max
|
|
43
|
+
lines += [{ text: "", color: @palette[:fg] }] * pad
|
|
44
|
+
|
|
45
|
+
argv = ["-size", "#{@width}x#{@height}", "xc:#{@palette[:bg]}"]
|
|
46
|
+
argv += header_bar_args
|
|
47
|
+
y = HEADER_H + PAD_Y + char_h - (char_h * 0.28).round
|
|
48
|
+
lines.each do |line|
|
|
49
|
+
argv += line_args(line, y)
|
|
50
|
+
y += char_h
|
|
51
|
+
end
|
|
52
|
+
argv << out_path
|
|
53
|
+
|
|
54
|
+
run!(argv)
|
|
55
|
+
out_path
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
# Measures the font's per-character advance width, not a single
|
|
61
|
+
# glyph's ink width -- those differ (a glyph is typically narrower
|
|
62
|
+
# than its cell) and only the advance width is what lines up
|
|
63
|
+
# correctly when text is drawn as several separate -annotate calls,
|
|
64
|
+
# as syntax-highlighted spans are. Comparing two label: widths of
|
|
65
|
+
# different lengths and dividing by the difference in length
|
|
66
|
+
# cancels out the fixed left/right bearing "label:" adds around the
|
|
67
|
+
# text, isolating the true per-character advance.
|
|
68
|
+
SHORT_CALIB_LEN = 10
|
|
69
|
+
LONG_CALIB_LEN = 30
|
|
70
|
+
|
|
71
|
+
def calibrate!
|
|
72
|
+
Dir.mktmpdir do |dir|
|
|
73
|
+
short = File.join(dir, "short.png")
|
|
74
|
+
long = File.join(dir, "long.png")
|
|
75
|
+
run!(["-background", "none", "-fill", "black", "-font", @font,
|
|
76
|
+
"-pointsize", @config.font_size.to_s, "label:#{'0' * SHORT_CALIB_LEN}", short])
|
|
77
|
+
run!(["-background", "none", "-fill", "black", "-font", @font,
|
|
78
|
+
"-pointsize", @config.font_size.to_s, "label:#{'0' * LONG_CALIB_LEN}", long])
|
|
79
|
+
w_short, h = `identify -format "%w %h" #{Shellwords.escape(short)}`.split.map(&:to_i)
|
|
80
|
+
w_long, = `identify -format "%w %h" #{Shellwords.escape(long)}`.split.map(&:to_i)
|
|
81
|
+
raise RenderError, "could not calibrate font metrics for #{@font.inspect}" if w_short.to_i <= 0 || w_long.to_i <= 0
|
|
82
|
+
|
|
83
|
+
@char_w = ((w_long - w_short).to_f / (LONG_CALIB_LEN - SHORT_CALIB_LEN)).round
|
|
84
|
+
@char_h = (h * 1.35).round
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def header_bar_args
|
|
89
|
+
args = ["-fill", @palette[:header], "-draw", "rectangle 0,0 #{@width},#{HEADER_H}"]
|
|
90
|
+
cx = PAD_X
|
|
91
|
+
cy = HEADER_H / 2
|
|
92
|
+
[@palette[:dot_red], @palette[:dot_yellow], @palette[:dot_green]].each do |color|
|
|
93
|
+
args += ["-fill", color, "-draw", "circle #{cx},#{cy} #{cx + DOT_R},#{cy}"]
|
|
94
|
+
cx += DOT_R * 3
|
|
95
|
+
end
|
|
96
|
+
args += ["-fill", @palette[:muted], "-font", @font, "-pointsize", (@config.font_size * 0.7).round.to_s,
|
|
97
|
+
"-gravity", "North", "-annotate", "+0+#{(HEADER_H - @config.font_size * 0.7) / 2 - 2}", escape_annotate(@config.title),
|
|
98
|
+
"-gravity", "NorthWest"]
|
|
99
|
+
args
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# A line is either a single-color {text:, color:} (prose) or a
|
|
103
|
+
# multi-color {spans: [{text:, color:}, ...]} (a highlighted code
|
|
104
|
+
# line). Spans are drawn as consecutive -annotate calls, each
|
|
105
|
+
# offset by the fixed glyph width times the characters already
|
|
106
|
+
# placed -- exact because the font is monospace.
|
|
107
|
+
def line_args(line, y)
|
|
108
|
+
if line[:spans]
|
|
109
|
+
x = PAD_X
|
|
110
|
+
line[:spans].flat_map do |span|
|
|
111
|
+
args = annotate_args(span[:text], span[:color], x, y)
|
|
112
|
+
x += char_w * span[:text].length
|
|
113
|
+
args
|
|
114
|
+
end
|
|
115
|
+
else
|
|
116
|
+
annotate_args(line[:text], line[:color], PAD_X, y)
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# ImageMagick's -annotate silently trims leading whitespace off the
|
|
121
|
+
# text it's given -- a mixed span like " sum(" would draw as "sum("
|
|
122
|
+
# flush against whatever x we asked for, one glyph closer than
|
|
123
|
+
# intended, colliding with the previous span. Since we position
|
|
124
|
+
# every span explicitly rather than relying on IM's own text flow,
|
|
125
|
+
# the fix is to strip the whitespace ourselves and shift x by
|
|
126
|
+
# however many characters we stripped, so IM never sees (and can't
|
|
127
|
+
# silently eat) leading whitespace in what it draws. A whitespace-
|
|
128
|
+
# only span -- e.g. a code line's leading indent, its own span
|
|
129
|
+
# since it's a different color to what follows -- draws nothing at
|
|
130
|
+
# all, correctly, since the canvas is already blank there.
|
|
131
|
+
def annotate_args(text, color, x, y)
|
|
132
|
+
leading = text[/\A\s*/].length
|
|
133
|
+
visible = text.strip
|
|
134
|
+
return [] if visible.empty?
|
|
135
|
+
|
|
136
|
+
["-fill", color, "-font", @font, "-pointsize", @config.font_size.to_s,
|
|
137
|
+
"-annotate", "+#{x + char_w * leading}+#{y}", escape_annotate(visible)]
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# ImageMagick's -annotate treats a literal backslash in the text as
|
|
141
|
+
# the start of an escape (e.g. a bare "\n" becomes a real line
|
|
142
|
+
# break), independent of our own word-wrap. Text like a shell
|
|
143
|
+
# `"...\n..."` argument that never got interpreted as a real
|
|
144
|
+
# newline would otherwise grow extra lines IM knows about but our
|
|
145
|
+
# canvas-height math (based on real "\n" splits) doesn't -- pushing
|
|
146
|
+
# later lines down until they overflow and overlap. Doubling
|
|
147
|
+
# backslashes makes IM render them literally instead.
|
|
148
|
+
def escape_annotate(text)
|
|
149
|
+
text.gsub("\\") { "\\\\" }
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Builds the wrapped, cursor-annotated lines for a state, without
|
|
153
|
+
# the trailing pad-to-total_lines step (callers that need a fixed
|
|
154
|
+
# canvas height add that themselves).
|
|
155
|
+
def wrap_body(state)
|
|
156
|
+
body = wrap_text("#{@config.label}#{state.prompt_text}", @config.cols)
|
|
157
|
+
.map { |l| { text: l, color: @palette[:prompt] } }
|
|
158
|
+
|
|
159
|
+
case state.phase
|
|
160
|
+
when :typing_prompt
|
|
161
|
+
append_cursor!(body, state.cursor_on)
|
|
162
|
+
return body
|
|
163
|
+
when :thinking
|
|
164
|
+
body << { text: "", color: @palette[:fg] }
|
|
165
|
+
dots = "." * state.dot_count
|
|
166
|
+
body << { text: "#{@config.thinking_label}#{dots}", color: @palette[:muted] }
|
|
167
|
+
append_cursor!(body, state.cursor_on)
|
|
168
|
+
return body
|
|
169
|
+
when :reasoning
|
|
170
|
+
body << { text: "", color: @palette[:fg] }
|
|
171
|
+
body << { text: @config.thinking_label, color: @palette[:muted] }
|
|
172
|
+
body += code_aware_lines(state.reasoning_text, @config.cols - 2, @palette[:muted])
|
|
173
|
+
.map { |l| indent_line(l, " ") }
|
|
174
|
+
append_cursor!(body, state.cursor_on)
|
|
175
|
+
return body
|
|
176
|
+
else
|
|
177
|
+
body << { text: "", color: @palette[:fg] }
|
|
178
|
+
body += code_aware_lines(state.response_text, @config.cols, @palette[:fg])
|
|
179
|
+
append_cursor!(body, state.cursor_on)
|
|
180
|
+
return body
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def append_cursor!(body, cursor_on)
|
|
185
|
+
return unless cursor_on
|
|
186
|
+
|
|
187
|
+
last = body.last
|
|
188
|
+
if last[:spans]
|
|
189
|
+
last[:spans] << { text: @config.cursor_char, color: @palette[:fg] }
|
|
190
|
+
else
|
|
191
|
+
last[:text] = "#{last[:text]}#{@config.cursor_char}"
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def indent_line(line, prefix)
|
|
196
|
+
if line[:spans]
|
|
197
|
+
{ spans: [{ text: prefix, color: @palette[:muted] }] + line[:spans] }
|
|
198
|
+
else
|
|
199
|
+
{ text: "#{prefix}#{line[:text]}", color: line[:color] }
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def wrap_text(str, cols)
|
|
204
|
+
str.split("\n", -1).flat_map { |para| wrap_paragraph(para, cols) }
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
FENCE = /\A\s{0,3}(?:```|~~~)/
|
|
208
|
+
# A trailing line consisting of only 1-2 backticks/tildes could
|
|
209
|
+
# still grow into a real fence marker on the next frame (relevant
|
|
210
|
+
# with --unit char, where streaming can stop mid-marker). Since
|
|
211
|
+
# only the very last line of a partially-revealed text can be
|
|
212
|
+
# incomplete like this, it's held back for a frame rather than
|
|
213
|
+
# guessed at -- guessing wrong either toggles code state a
|
|
214
|
+
# character early or (worse) briefly counts an extra line that the
|
|
215
|
+
# fully-revealed text, which sized the canvas, never has.
|
|
216
|
+
PENDING_FENCE = /\A\s{0,3}(?:`{1,2}|~{1,2})\z/
|
|
217
|
+
|
|
218
|
+
# Walks text line by line, toggling in/out of "code" on fence
|
|
219
|
+
# markers (```/~~~). Fence marker lines themselves are dropped from
|
|
220
|
+
# the output -- the fence is how the source marks the block, not
|
|
221
|
+
# something meant to show up in a rendered chat transcript. Prose
|
|
222
|
+
# lines word-wrap and render in a single color, same as before;
|
|
223
|
+
# code lines are split into syntax-highlighted spans. Because this
|
|
224
|
+
# runs on whatever prefix of the text has streamed in so far, an
|
|
225
|
+
# unterminated trailing fence is simply treated as "still in code".
|
|
226
|
+
def code_aware_lines(text, cols, prose_color)
|
|
227
|
+
in_code = false
|
|
228
|
+
out = []
|
|
229
|
+
raw_lines = text.split("\n", -1)
|
|
230
|
+
raw_lines.pop if PENDING_FENCE.match?(raw_lines.last.to_s)
|
|
231
|
+
|
|
232
|
+
raw_lines.each do |raw|
|
|
233
|
+
if raw =~ FENCE
|
|
234
|
+
in_code = !in_code
|
|
235
|
+
next
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
out.concat(in_code ? wrap_code_line(raw, cols) : wrap_paragraph(raw, cols).map { |l| { text: l, color: prose_color } })
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
out << { text: "", color: prose_color } if out.empty?
|
|
242
|
+
out
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def wrap_code_line(raw, cols)
|
|
246
|
+
chunks = raw.empty? ? [""] : raw.chars.each_slice([cols, 1].max).map(&:join)
|
|
247
|
+
chunks.map { |chunk| { spans: Highlight.spans(chunk, @palette) } }
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def wrap_paragraph(line, cols)
|
|
251
|
+
return [""] if line.empty?
|
|
252
|
+
|
|
253
|
+
words = line.scan(/\S+\s*|\s+/)
|
|
254
|
+
lines = []
|
|
255
|
+
current = +""
|
|
256
|
+
words.each do |w|
|
|
257
|
+
if !current.empty? && (current.length + w.length) > cols
|
|
258
|
+
lines << current.rstrip
|
|
259
|
+
current = w.lstrip
|
|
260
|
+
else
|
|
261
|
+
current << w
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
lines << current.rstrip unless current.empty?
|
|
265
|
+
lines
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
def run!(argv)
|
|
269
|
+
_out, err, status = Open3.capture3(self.class.convert_binary, *argv)
|
|
270
|
+
raise RenderError, "ImageMagick failed: #{err}" unless status.success?
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TokenReel
|
|
4
|
+
# Parses a single Markdown file into the named sections a render
|
|
5
|
+
# needs -- prompt, reasoning, and response -- so a whole exchange can
|
|
6
|
+
# be written and version-controlled as one file instead of juggling
|
|
7
|
+
# separate --prompt/--response flags.
|
|
8
|
+
#
|
|
9
|
+
# Recognized headings (any level, case-insensitive; content before
|
|
10
|
+
# the first recognized heading is ignored):
|
|
11
|
+
#
|
|
12
|
+
# # Prompt | User | Input | Question
|
|
13
|
+
# # Reasoning | Thinking | Thought
|
|
14
|
+
# # Output | Response | Answer | Assistant
|
|
15
|
+
#
|
|
16
|
+
# A heading is only recognized outside of a fenced code block, so a
|
|
17
|
+
# shell/Python comment like "# Output" inside a ```fence``` never
|
|
18
|
+
# gets mistaken for a section header. Fence markers themselves are
|
|
19
|
+
# kept in the extracted section text -- Renderer is what looks for
|
|
20
|
+
# them, so this works the same whether the text came from a Markdown
|
|
21
|
+
# file or a plain --response string.
|
|
22
|
+
#
|
|
23
|
+
# `.template` returns a starter file with all three headings already
|
|
24
|
+
# in place, for `token_reel --init-markdown` to write out.
|
|
25
|
+
module Script
|
|
26
|
+
SECTION_ALIASES = {
|
|
27
|
+
"prompt" => :prompt, "user" => :prompt, "input" => :prompt, "question" => :prompt,
|
|
28
|
+
"reasoning" => :reasoning, "thinking" => :reasoning, "thought" => :reasoning,
|
|
29
|
+
"output" => :response, "response" => :response, "answer" => :response, "assistant" => :response
|
|
30
|
+
}.freeze
|
|
31
|
+
|
|
32
|
+
HEADING = /\A\s{0,3}\#{1,6}\s+(.+?)\s*\z/
|
|
33
|
+
FENCE = /\A\s{0,3}(?:```|~~~)/
|
|
34
|
+
|
|
35
|
+
TEMPLATE = <<~MD
|
|
36
|
+
## Prompt
|
|
37
|
+
|
|
38
|
+
Replace this with the prompt text (shown as typed input).
|
|
39
|
+
|
|
40
|
+
## Reasoning
|
|
41
|
+
|
|
42
|
+
Optional -- delete this whole section if you don't want a "thinking"
|
|
43
|
+
trace. Replace this with the reasoning text; it streams first, then
|
|
44
|
+
is replaced by the response.
|
|
45
|
+
|
|
46
|
+
## Output
|
|
47
|
+
|
|
48
|
+
Replace this with the response text (streamed as output). Code
|
|
49
|
+
fences work here too and are syntax-highlighted:
|
|
50
|
+
|
|
51
|
+
```ruby
|
|
52
|
+
def hello
|
|
53
|
+
puts "hi"
|
|
54
|
+
end
|
|
55
|
+
```
|
|
56
|
+
MD
|
|
57
|
+
|
|
58
|
+
# A starter Markdown file with the three recognized headings
|
|
59
|
+
# already in place, ready to fill in and pass to `-m`/`--markdown`.
|
|
60
|
+
def self.template
|
|
61
|
+
TEMPLATE
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Returns { prompt: "...", reasoning: "...", response: "..." },
|
|
65
|
+
# omitting keys whose section was absent from the file.
|
|
66
|
+
def self.parse(text)
|
|
67
|
+
sections = Hash.new { |h, k| h[k] = +"" }
|
|
68
|
+
current = nil
|
|
69
|
+
in_fence = false
|
|
70
|
+
|
|
71
|
+
text.each_line do |line|
|
|
72
|
+
stripped = line.chomp
|
|
73
|
+
in_fence = !in_fence if stripped =~ FENCE
|
|
74
|
+
|
|
75
|
+
if !in_fence && (m = HEADING.match(stripped)) && (key = SECTION_ALIASES[m[1].downcase])
|
|
76
|
+
current = key
|
|
77
|
+
next
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
sections[current] << line if current
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
sections.transform_values(&:strip)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TokenReel
|
|
4
|
+
module Theme
|
|
5
|
+
PALETTES = {
|
|
6
|
+
dark: {
|
|
7
|
+
bg: "#0d1117", header: "#161b22", border: "#30363d",
|
|
8
|
+
fg: "#c9d1d9", prompt: "#7ee787", muted: "#8b949e",
|
|
9
|
+
dot_red: "#ff5f56", dot_yellow: "#ffbd2e", dot_green: "#27c93f",
|
|
10
|
+
syn_keyword: "#ff7b72", syn_string: "#a5d6ff", syn_number: "#79c0ff", syn_comment: "#8b949e"
|
|
11
|
+
},
|
|
12
|
+
matrix: {
|
|
13
|
+
bg: "#000000", header: "#000000", border: "#003300",
|
|
14
|
+
fg: "#00ff41", prompt: "#00ff41", muted: "#008f11",
|
|
15
|
+
dot_red: "#003300", dot_yellow: "#005500", dot_green: "#00ff41",
|
|
16
|
+
syn_keyword: "#00ffae", syn_string: "#7fff7f", syn_number: "#39ff14", syn_comment: "#008f11"
|
|
17
|
+
},
|
|
18
|
+
light: {
|
|
19
|
+
bg: "#ffffff", header: "#f0f0f0", border: "#d0d0d0",
|
|
20
|
+
fg: "#24292f", prompt: "#116329", muted: "#6e7781",
|
|
21
|
+
dot_red: "#ff5f56", dot_yellow: "#ffbd2e", dot_green: "#27c93f",
|
|
22
|
+
syn_keyword: "#cf222e", syn_string: "#0a3069", syn_number: "#0550ae", syn_comment: "#6e7781"
|
|
23
|
+
},
|
|
24
|
+
solarized: {
|
|
25
|
+
bg: "#002b36", header: "#073642", border: "#586e75",
|
|
26
|
+
fg: "#eee8d5", prompt: "#2aa198", muted: "#93a1a1",
|
|
27
|
+
dot_red: "#dc322f", dot_yellow: "#b58900", dot_green: "#859900",
|
|
28
|
+
syn_keyword: "#268bd2", syn_string: "#859900", syn_number: "#cb4b16", syn_comment: "#93a1a1"
|
|
29
|
+
}
|
|
30
|
+
}.freeze
|
|
31
|
+
|
|
32
|
+
def self.fetch(name)
|
|
33
|
+
PALETTES.fetch(name.to_sym) do
|
|
34
|
+
raise ConfigError, "unknown theme #{name.inspect} (valid: #{PALETTES.keys.join(', ')})"
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TokenReel
|
|
4
|
+
# A snapshot of the screen at a given moment: how much of the prompt,
|
|
5
|
+
# reasoning, and response are visible, which "phase" we're in, and
|
|
6
|
+
# whether the cursor happens to be in its "on" blink state.
|
|
7
|
+
State = Struct.new(:phase, :prompt_text, :reasoning_text, :response_text, :cursor_on, :dot_count, keyword_init: true) do
|
|
8
|
+
# Identical signatures render to the identical frame, so the
|
|
9
|
+
# sampler can skip re-rendering and just extend the previous
|
|
10
|
+
# frame's delay instead.
|
|
11
|
+
def signature
|
|
12
|
+
[phase, prompt_text, reasoning_text, response_text, cursor_on, dot_count]
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Pure function of time -> State. Doesn't touch the filesystem or
|
|
17
|
+
# ImageMagick at all, which makes it cheap to sample as densely as
|
|
18
|
+
# we like when building the frame list.
|
|
19
|
+
class Timeline
|
|
20
|
+
BLINK_HZ = 2.0 # cursor toggles this many times per second
|
|
21
|
+
DOT_INTERVAL = 0.35 # seconds between "thinking..." dot ticks
|
|
22
|
+
|
|
23
|
+
attr_reader :prompt_tokens, :reasoning_tokens, :response_tokens,
|
|
24
|
+
:prompt_full, :reasoning_full, :response_full,
|
|
25
|
+
:prompt_done_t, :thinking_end_t, :reasoning_end_t, :stream_end_t, :duration
|
|
26
|
+
|
|
27
|
+
def initialize(config)
|
|
28
|
+
@config = config
|
|
29
|
+
@prompt_tokens = Tokenizer.tokenize(config.prompt, config.unit)
|
|
30
|
+
@reasoning_tokens = Tokenizer.tokenize(config.reasoning, config.unit)
|
|
31
|
+
@response_tokens = Tokenizer.tokenize(config.response, config.unit)
|
|
32
|
+
@prompt_full = prompt_tokens.join
|
|
33
|
+
@reasoning_full = reasoning_tokens.join
|
|
34
|
+
@response_full = response_tokens.join
|
|
35
|
+
|
|
36
|
+
@prompt_interval = config.prompt_tps.to_f.positive? ? 1.0 / config.prompt_tps : 0
|
|
37
|
+
@reasoning_interval = config.reasoning_tps.to_f.positive? ? 1.0 / config.reasoning_tps : 1.0 / config.tps
|
|
38
|
+
@response_interval = 1.0 / config.tps
|
|
39
|
+
|
|
40
|
+
@prompt_done_t = @prompt_interval * prompt_tokens.size
|
|
41
|
+
@thinking_end_t = prompt_done_t + config.ttft.to_f
|
|
42
|
+
@reasoning_end_t = thinking_end_t + @reasoning_interval * reasoning_tokens.size
|
|
43
|
+
@stream_end_t = reasoning_end_t + @response_interval * response_tokens.size
|
|
44
|
+
@duration = stream_end_t + config.hold.to_f
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def state_at(t)
|
|
48
|
+
t = t.clamp(0, duration)
|
|
49
|
+
|
|
50
|
+
if t < prompt_done_t
|
|
51
|
+
n = @prompt_interval.positive? ? (t / @prompt_interval).floor : prompt_tokens.size
|
|
52
|
+
State.new(
|
|
53
|
+
phase: :typing_prompt,
|
|
54
|
+
prompt_text: prompt_tokens[0...n].join,
|
|
55
|
+
reasoning_text: "",
|
|
56
|
+
response_text: "",
|
|
57
|
+
cursor_on: blink_on?(t),
|
|
58
|
+
dot_count: 0
|
|
59
|
+
)
|
|
60
|
+
elsif t < thinking_end_t
|
|
61
|
+
elapsed = t - prompt_done_t
|
|
62
|
+
State.new(
|
|
63
|
+
phase: :thinking,
|
|
64
|
+
prompt_text: prompt_full,
|
|
65
|
+
reasoning_text: "",
|
|
66
|
+
response_text: "",
|
|
67
|
+
cursor_on: blink_on?(t),
|
|
68
|
+
dot_count: ((elapsed / DOT_INTERVAL).to_i % 4)
|
|
69
|
+
)
|
|
70
|
+
elsif t < reasoning_end_t
|
|
71
|
+
elapsed = t - thinking_end_t
|
|
72
|
+
n = @reasoning_interval.positive? ? (elapsed / @reasoning_interval).floor : reasoning_tokens.size
|
|
73
|
+
n = n.clamp(0, reasoning_tokens.size)
|
|
74
|
+
State.new(
|
|
75
|
+
phase: :reasoning,
|
|
76
|
+
prompt_text: prompt_full,
|
|
77
|
+
reasoning_text: reasoning_tokens[0...n].join,
|
|
78
|
+
response_text: "",
|
|
79
|
+
cursor_on: blink_on?(t),
|
|
80
|
+
dot_count: 0
|
|
81
|
+
)
|
|
82
|
+
else
|
|
83
|
+
elapsed = t - reasoning_end_t
|
|
84
|
+
n = @response_interval.positive? ? (elapsed / @response_interval).floor : response_tokens.size
|
|
85
|
+
n = n.clamp(0, response_tokens.size)
|
|
86
|
+
done = n >= response_tokens.size
|
|
87
|
+
State.new(
|
|
88
|
+
phase: done ? :done : :streaming,
|
|
89
|
+
prompt_text: prompt_full,
|
|
90
|
+
reasoning_text: "",
|
|
91
|
+
response_text: response_tokens[0...n].join,
|
|
92
|
+
cursor_on: blink_on?(t),
|
|
93
|
+
dot_count: 0
|
|
94
|
+
)
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# The fully-revealed, cursor-off state -- used to size the canvas
|
|
99
|
+
# so every frame in the GIF shares identical dimensions.
|
|
100
|
+
def final_state
|
|
101
|
+
State.new(phase: :done, prompt_text: prompt_full, reasoning_text: "", response_text: response_full,
|
|
102
|
+
cursor_on: false, dot_count: 0)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# The reasoning phase in full, cursor off -- used alongside
|
|
106
|
+
# final_state to size the canvas, since the reasoning trace (shown
|
|
107
|
+
# only while streaming, then replaced by the response) can be
|
|
108
|
+
# taller than the finished response.
|
|
109
|
+
def max_reasoning_state
|
|
110
|
+
State.new(phase: :reasoning, prompt_text: prompt_full, reasoning_text: reasoning_full, response_text: "",
|
|
111
|
+
cursor_on: false, dot_count: 0)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
private
|
|
115
|
+
|
|
116
|
+
def blink_on?(t)
|
|
117
|
+
(t * (2 * BLINK_HZ)).to_i.even?
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TokenReel
|
|
4
|
+
# Splits text into the chunks that get revealed one-by-one while
|
|
5
|
+
# streaming. Joining the tokens back together always reproduces the
|
|
6
|
+
# original string exactly.
|
|
7
|
+
module Tokenizer
|
|
8
|
+
def self.tokenize(text, unit)
|
|
9
|
+
return [] if text.nil? || text.empty?
|
|
10
|
+
|
|
11
|
+
case unit.to_sym
|
|
12
|
+
when :char
|
|
13
|
+
text.each_char.to_a
|
|
14
|
+
when :word
|
|
15
|
+
# "word" + any trailing whitespace travels together, so a token
|
|
16
|
+
# reveals as a whole word (closer to how LLM tokens/BPE chunks
|
|
17
|
+
# tend to land) instead of one raw character at a time.
|
|
18
|
+
text.scan(/\S+\s*|\s+/)
|
|
19
|
+
else
|
|
20
|
+
raise ConfigError, "unknown unit #{unit.inspect} (valid: word, char)"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
data/lib/token_reel.rb
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "shellwords"
|
|
4
|
+
|
|
5
|
+
require_relative "token_reel/version"
|
|
6
|
+
require_relative "token_reel/errors"
|
|
7
|
+
require_relative "token_reel/theme"
|
|
8
|
+
require_relative "token_reel/tokenizer"
|
|
9
|
+
require_relative "token_reel/script"
|
|
10
|
+
require_relative "token_reel/config"
|
|
11
|
+
require_relative "token_reel/timeline"
|
|
12
|
+
require_relative "token_reel/fonts"
|
|
13
|
+
require_relative "token_reel/highlight"
|
|
14
|
+
require_relative "token_reel/renderer"
|
|
15
|
+
require_relative "token_reel/gif_writer"
|
|
16
|
+
require_relative "token_reel/generator"
|
|
17
|
+
require_relative "token_reel/cli"
|
|
18
|
+
|
|
19
|
+
module TokenReel
|
|
20
|
+
# Convenience one-liner: TokenReel.generate(prompt: "...", response: "...", tps: 12)
|
|
21
|
+
def self.generate(**opts)
|
|
22
|
+
config = Config.new
|
|
23
|
+
opts.each do |k, v|
|
|
24
|
+
setter = "#{k}="
|
|
25
|
+
raise ConfigError, "unknown option #{k.inspect}" unless config.respond_to?(setter)
|
|
26
|
+
|
|
27
|
+
config.public_send(setter, v)
|
|
28
|
+
end
|
|
29
|
+
Generator.new(config).generate!
|
|
30
|
+
end
|
|
31
|
+
end
|
data/token_reel.gemspec
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lib/token_reel/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "token_reel"
|
|
7
|
+
spec.version = TokenReel::VERSION
|
|
8
|
+
spec.authors = ["Thomas Powell"]
|
|
9
|
+
spec.email = ["twilliampowell@gmail.com"]
|
|
10
|
+
spec.summary = "Render terminal-style GIFs of an LLM prompt/response, streamed at a chosen tokens/sec and time-to-first-token"
|
|
11
|
+
spec.description = <<~DESC
|
|
12
|
+
token_reel renders a "CLI demo"-style animated GIF of a prompt being
|
|
13
|
+
answered: the prompt appears, there's a configurable pause (time to
|
|
14
|
+
first token), and the response streams in word-by-word or
|
|
15
|
+
character-by-character at a configurable tokens/sec rate -- handy for
|
|
16
|
+
READMEs, blog posts, and talks that want to show off an LLM CLI
|
|
17
|
+
without shelling out to a real model (or a real screen recorder).
|
|
18
|
+
DESC
|
|
19
|
+
spec.homepage = "https://github.com/stringsn88keys/token_reel"
|
|
20
|
+
spec.license = "MIT"
|
|
21
|
+
spec.required_ruby_version = ">= 3.0"
|
|
22
|
+
|
|
23
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
|
24
|
+
spec.metadata["source_code_uri"] = spec.homepage
|
|
25
|
+
|
|
26
|
+
spec.files = Dir.chdir(__dir__) do
|
|
27
|
+
if system("git rev-parse --git-dir > /dev/null 2>&1")
|
|
28
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
|
29
|
+
else
|
|
30
|
+
Dir.glob("{lib,exe}/**/*", File::FNM_DOTMATCH).select { |f| File.file?(f) } +
|
|
31
|
+
%w[README.md token_reel.gemspec]
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
spec.bindir = "exe"
|
|
35
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
36
|
+
spec.require_paths = ["lib"]
|
|
37
|
+
|
|
38
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
|
39
|
+
spec.add_development_dependency "rspec", "~> 3.12"
|
|
40
|
+
end
|