token_reel 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TokenReel
4
+ # A small, dependency-free syntax colorizer for text inside fenced
5
+ # code blocks. It isn't a real per-language grammar -- just enough
6
+ # regex-based token classing (comments, strings, numbers, a shared
7
+ # keyword list spanning a handful of mainstream languages) to make
8
+ # demo code read as code instead of one flat color. The language tag
9
+ # on a fence (```ruby) is cosmetic; every fence is highlighted the
10
+ # same way.
11
+ module Highlight
12
+ KEYWORDS = %w[
13
+ def end class module function fn func return if elif elsif else
14
+ unless while until for do begin rescue ensure raise throw try
15
+ catch except finally break next continue yield case when switch
16
+ match default import export from require include use pub package
17
+ namespace let const var static final public private protected
18
+ abstract new self this super nil null none true false async await
19
+ int float double bool bool8 string char void byte long short
20
+ struct enum interface extends implements typeof instanceof
21
+ in of as with lambda del pass global nonlocal
22
+ ].freeze
23
+
24
+ # One token per iteration: a comment to end-of-line, a quoted
25
+ # string (single/double/backtick, backslash-escaped), an integer
26
+ # or float, an identifier/keyword, or a single other character
27
+ # (whitespace, punctuation, operators).
28
+ TOKEN = /
29
+ (?<comment>\#.*\z|\/\/.*\z)
30
+ |(?<string>"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`(?:[^`\\]|\\.)*`)
31
+ |(?<number>\b\d+(?:\.\d+)?\b)
32
+ |(?<word>[A-Za-z_][A-Za-z0-9_]*)
33
+ |(?<other>.)
34
+ /x
35
+
36
+ # Splits one line of code into color-tagged spans. Adjacent tokens
37
+ # that land on the same color are merged, so a run of plain text
38
+ # becomes one span instead of one per character.
39
+ def self.spans(line, palette)
40
+ out = []
41
+ line.scan(TOKEN) do
42
+ m = Regexp.last_match
43
+ text, color = classify(m, palette)
44
+ if out.any? && out.last[:color] == color
45
+ out.last[:text] += text
46
+ else
47
+ out << { text: text, color: color }
48
+ end
49
+ end
50
+ out
51
+ end
52
+
53
+ def self.classify(m, palette)
54
+ return [m[:comment], palette[:syn_comment]] if m[:comment]
55
+ return [m[:string], palette[:syn_string]] if m[:string]
56
+ return [m[:number], palette[:syn_number]] if m[:number]
57
+ return [m[:word], KEYWORDS.include?(m[:word]) ? palette[:syn_keyword] : palette[:fg]] if m[:word]
58
+
59
+ [m[:other], palette[:fg]]
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,273 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "open3"
4
+
5
+ module TokenReel
6
+ # Turns a Timeline::State into a terminal-window PNG. Text layout is
7
+ # done in Ruby (word wrap, cursor placement); ImageMagick just draws
8
+ # the rectangles and glyphs we tell it to.
9
+ class Renderer
10
+ HEADER_H = 40
11
+ PAD_X = 18
12
+ PAD_Y = 14
13
+ DOT_R = 6
14
+
15
+ attr_reader :char_w, :char_h, :total_lines
16
+
17
+ def self.convert_binary
18
+ @convert_binary ||= %w[magick convert].find { |bin| system("which #{bin} > /dev/null 2>&1") } ||
19
+ raise(RenderError, "ImageMagick not found: install `imagemagick` (needs `convert` or `magick` on PATH)")
20
+ end
21
+
22
+ def initialize(config, timeline)
23
+ @config = config
24
+ @timeline = timeline
25
+ @palette = Theme.fetch(config.theme)
26
+ @font = Fonts.resolve!(config.font)
27
+ calibrate!
28
+ # The window is a fixed config.rows lines tall, like a real
29
+ # console -- every frame shares this height regardless of how
30
+ # much text it holds.
31
+ @total_lines = @config.rows
32
+ @width = @config.cols * char_w + PAD_X * 2
33
+ @height = HEADER_H + PAD_Y * 2 + total_lines * char_h
34
+ end
35
+
36
+ def render(state, out_path)
37
+ lines = wrap_body(state)
38
+ # Once a frame's content outgrows the window, scroll: keep only
39
+ # the most recent total_lines lines, same as a real terminal
40
+ # dropping its oldest lines off the top.
41
+ lines = lines.last(total_lines) if lines.size > total_lines
42
+ pad = [total_lines - lines.size, 0].max
43
+ lines += [{ text: "", color: @palette[:fg] }] * pad
44
+
45
+ argv = ["-size", "#{@width}x#{@height}", "xc:#{@palette[:bg]}"]
46
+ argv += header_bar_args
47
+ y = HEADER_H + PAD_Y + char_h - (char_h * 0.28).round
48
+ lines.each do |line|
49
+ argv += line_args(line, y)
50
+ y += char_h
51
+ end
52
+ argv << out_path
53
+
54
+ run!(argv)
55
+ out_path
56
+ end
57
+
58
+ private
59
+
60
+ # Measures the font's per-character advance width, not a single
61
+ # glyph's ink width -- those differ (a glyph is typically narrower
62
+ # than its cell) and only the advance width is what lines up
63
+ # correctly when text is drawn as several separate -annotate calls,
64
+ # as syntax-highlighted spans are. Comparing two label: widths of
65
+ # different lengths and dividing by the difference in length
66
+ # cancels out the fixed left/right bearing "label:" adds around the
67
+ # text, isolating the true per-character advance.
68
+ SHORT_CALIB_LEN = 10
69
+ LONG_CALIB_LEN = 30
70
+
71
+ def calibrate!
72
+ Dir.mktmpdir do |dir|
73
+ short = File.join(dir, "short.png")
74
+ long = File.join(dir, "long.png")
75
+ run!(["-background", "none", "-fill", "black", "-font", @font,
76
+ "-pointsize", @config.font_size.to_s, "label:#{'0' * SHORT_CALIB_LEN}", short])
77
+ run!(["-background", "none", "-fill", "black", "-font", @font,
78
+ "-pointsize", @config.font_size.to_s, "label:#{'0' * LONG_CALIB_LEN}", long])
79
+ w_short, h = `identify -format "%w %h" #{Shellwords.escape(short)}`.split.map(&:to_i)
80
+ w_long, = `identify -format "%w %h" #{Shellwords.escape(long)}`.split.map(&:to_i)
81
+ raise RenderError, "could not calibrate font metrics for #{@font.inspect}" if w_short.to_i <= 0 || w_long.to_i <= 0
82
+
83
+ @char_w = ((w_long - w_short).to_f / (LONG_CALIB_LEN - SHORT_CALIB_LEN)).round
84
+ @char_h = (h * 1.35).round
85
+ end
86
+ end
87
+
88
+ def header_bar_args
89
+ args = ["-fill", @palette[:header], "-draw", "rectangle 0,0 #{@width},#{HEADER_H}"]
90
+ cx = PAD_X
91
+ cy = HEADER_H / 2
92
+ [@palette[:dot_red], @palette[:dot_yellow], @palette[:dot_green]].each do |color|
93
+ args += ["-fill", color, "-draw", "circle #{cx},#{cy} #{cx + DOT_R},#{cy}"]
94
+ cx += DOT_R * 3
95
+ end
96
+ args += ["-fill", @palette[:muted], "-font", @font, "-pointsize", (@config.font_size * 0.7).round.to_s,
97
+ "-gravity", "North", "-annotate", "+0+#{(HEADER_H - @config.font_size * 0.7) / 2 - 2}", escape_annotate(@config.title),
98
+ "-gravity", "NorthWest"]
99
+ args
100
+ end
101
+
102
+ # A line is either a single-color {text:, color:} (prose) or a
103
+ # multi-color {spans: [{text:, color:}, ...]} (a highlighted code
104
+ # line). Spans are drawn as consecutive -annotate calls, each
105
+ # offset by the fixed glyph width times the characters already
106
+ # placed -- exact because the font is monospace.
107
+ def line_args(line, y)
108
+ if line[:spans]
109
+ x = PAD_X
110
+ line[:spans].flat_map do |span|
111
+ args = annotate_args(span[:text], span[:color], x, y)
112
+ x += char_w * span[:text].length
113
+ args
114
+ end
115
+ else
116
+ annotate_args(line[:text], line[:color], PAD_X, y)
117
+ end
118
+ end
119
+
120
+ # ImageMagick's -annotate silently trims leading whitespace off the
121
+ # text it's given -- a mixed span like " sum(" would draw as "sum("
122
+ # flush against whatever x we asked for, one glyph closer than
123
+ # intended, colliding with the previous span. Since we position
124
+ # every span explicitly rather than relying on IM's own text flow,
125
+ # the fix is to strip the whitespace ourselves and shift x by
126
+ # however many characters we stripped, so IM never sees (and can't
127
+ # silently eat) leading whitespace in what it draws. A whitespace-
128
+ # only span -- e.g. a code line's leading indent, its own span
129
+ # since it's a different color to what follows -- draws nothing at
130
+ # all, correctly, since the canvas is already blank there.
131
+ def annotate_args(text, color, x, y)
132
+ leading = text[/\A\s*/].length
133
+ visible = text.strip
134
+ return [] if visible.empty?
135
+
136
+ ["-fill", color, "-font", @font, "-pointsize", @config.font_size.to_s,
137
+ "-annotate", "+#{x + char_w * leading}+#{y}", escape_annotate(visible)]
138
+ end
139
+
140
+ # ImageMagick's -annotate treats a literal backslash in the text as
141
+ # the start of an escape (e.g. a bare "\n" becomes a real line
142
+ # break), independent of our own word-wrap. Text like a shell
143
+ # `"...\n..."` argument that never got interpreted as a real
144
+ # newline would otherwise grow extra lines IM knows about but our
145
+ # canvas-height math (based on real "\n" splits) doesn't -- pushing
146
+ # later lines down until they overflow and overlap. Doubling
147
+ # backslashes makes IM render them literally instead.
148
+ def escape_annotate(text)
149
+ text.gsub("\\") { "\\\\" }
150
+ end
151
+
152
+ # Builds the wrapped, cursor-annotated lines for a state, without
153
+ # the trailing pad-to-total_lines step (callers that need a fixed
154
+ # canvas height add that themselves).
155
+ def wrap_body(state)
156
+ body = wrap_text("#{@config.label}#{state.prompt_text}", @config.cols)
157
+ .map { |l| { text: l, color: @palette[:prompt] } }
158
+
159
+ case state.phase
160
+ when :typing_prompt
161
+ append_cursor!(body, state.cursor_on)
162
+ return body
163
+ when :thinking
164
+ body << { text: "", color: @palette[:fg] }
165
+ dots = "." * state.dot_count
166
+ body << { text: "#{@config.thinking_label}#{dots}", color: @palette[:muted] }
167
+ append_cursor!(body, state.cursor_on)
168
+ return body
169
+ when :reasoning
170
+ body << { text: "", color: @palette[:fg] }
171
+ body << { text: @config.thinking_label, color: @palette[:muted] }
172
+ body += code_aware_lines(state.reasoning_text, @config.cols - 2, @palette[:muted])
173
+ .map { |l| indent_line(l, " ") }
174
+ append_cursor!(body, state.cursor_on)
175
+ return body
176
+ else
177
+ body << { text: "", color: @palette[:fg] }
178
+ body += code_aware_lines(state.response_text, @config.cols, @palette[:fg])
179
+ append_cursor!(body, state.cursor_on)
180
+ return body
181
+ end
182
+ end
183
+
184
+ def append_cursor!(body, cursor_on)
185
+ return unless cursor_on
186
+
187
+ last = body.last
188
+ if last[:spans]
189
+ last[:spans] << { text: @config.cursor_char, color: @palette[:fg] }
190
+ else
191
+ last[:text] = "#{last[:text]}#{@config.cursor_char}"
192
+ end
193
+ end
194
+
195
+ def indent_line(line, prefix)
196
+ if line[:spans]
197
+ { spans: [{ text: prefix, color: @palette[:muted] }] + line[:spans] }
198
+ else
199
+ { text: "#{prefix}#{line[:text]}", color: line[:color] }
200
+ end
201
+ end
202
+
203
+ def wrap_text(str, cols)
204
+ str.split("\n", -1).flat_map { |para| wrap_paragraph(para, cols) }
205
+ end
206
+
207
+ FENCE = /\A\s{0,3}(?:```|~~~)/
208
+ # A trailing line consisting of only 1-2 backticks/tildes could
209
+ # still grow into a real fence marker on the next frame (relevant
210
+ # with --unit char, where streaming can stop mid-marker). Since
211
+ # only the very last line of a partially-revealed text can be
212
+ # incomplete like this, it's held back for a frame rather than
213
+ # guessed at -- guessing wrong either toggles code state a
214
+ # character early or (worse) briefly counts an extra line that the
215
+ # fully-revealed text, which sized the canvas, never has.
216
+ PENDING_FENCE = /\A\s{0,3}(?:`{1,2}|~{1,2})\z/
217
+
218
+ # Walks text line by line, toggling in/out of "code" on fence
219
+ # markers (```/~~~). Fence marker lines themselves are dropped from
220
+ # the output -- the fence is how the source marks the block, not
221
+ # something meant to show up in a rendered chat transcript. Prose
222
+ # lines word-wrap and render in a single color, same as before;
223
+ # code lines are split into syntax-highlighted spans. Because this
224
+ # runs on whatever prefix of the text has streamed in so far, an
225
+ # unterminated trailing fence is simply treated as "still in code".
226
+ def code_aware_lines(text, cols, prose_color)
227
+ in_code = false
228
+ out = []
229
+ raw_lines = text.split("\n", -1)
230
+ raw_lines.pop if PENDING_FENCE.match?(raw_lines.last.to_s)
231
+
232
+ raw_lines.each do |raw|
233
+ if raw =~ FENCE
234
+ in_code = !in_code
235
+ next
236
+ end
237
+
238
+ out.concat(in_code ? wrap_code_line(raw, cols) : wrap_paragraph(raw, cols).map { |l| { text: l, color: prose_color } })
239
+ end
240
+
241
+ out << { text: "", color: prose_color } if out.empty?
242
+ out
243
+ end
244
+
245
+ def wrap_code_line(raw, cols)
246
+ chunks = raw.empty? ? [""] : raw.chars.each_slice([cols, 1].max).map(&:join)
247
+ chunks.map { |chunk| { spans: Highlight.spans(chunk, @palette) } }
248
+ end
249
+
250
+ def wrap_paragraph(line, cols)
251
+ return [""] if line.empty?
252
+
253
+ words = line.scan(/\S+\s*|\s+/)
254
+ lines = []
255
+ current = +""
256
+ words.each do |w|
257
+ if !current.empty? && (current.length + w.length) > cols
258
+ lines << current.rstrip
259
+ current = w.lstrip
260
+ else
261
+ current << w
262
+ end
263
+ end
264
+ lines << current.rstrip unless current.empty?
265
+ lines
266
+ end
267
+
268
+ def run!(argv)
269
+ _out, err, status = Open3.capture3(self.class.convert_binary, *argv)
270
+ raise RenderError, "ImageMagick failed: #{err}" unless status.success?
271
+ end
272
+ end
273
+ end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TokenReel
4
+ # Parses a single Markdown file into the named sections a render
5
+ # needs -- prompt, reasoning, and response -- so a whole exchange can
6
+ # be written and version-controlled as one file instead of juggling
7
+ # separate --prompt/--response flags.
8
+ #
9
+ # Recognized headings (any level, case-insensitive; content before
10
+ # the first recognized heading is ignored):
11
+ #
12
+ # # Prompt | User | Input | Question
13
+ # # Reasoning | Thinking | Thought
14
+ # # Output | Response | Answer | Assistant
15
+ #
16
+ # A heading is only recognized outside of a fenced code block, so a
17
+ # shell/Python comment like "# Output" inside a ```fence``` never
18
+ # gets mistaken for a section header. Fence markers themselves are
19
+ # kept in the extracted section text -- Renderer is what looks for
20
+ # them, so this works the same whether the text came from a Markdown
21
+ # file or a plain --response string.
22
+ #
23
+ # `.template` returns a starter file with all three headings already
24
+ # in place, for `token_reel --init-markdown` to write out.
25
+ module Script
26
+ SECTION_ALIASES = {
27
+ "prompt" => :prompt, "user" => :prompt, "input" => :prompt, "question" => :prompt,
28
+ "reasoning" => :reasoning, "thinking" => :reasoning, "thought" => :reasoning,
29
+ "output" => :response, "response" => :response, "answer" => :response, "assistant" => :response
30
+ }.freeze
31
+
32
+ HEADING = /\A\s{0,3}\#{1,6}\s+(.+?)\s*\z/
33
+ FENCE = /\A\s{0,3}(?:```|~~~)/
34
+
35
+ TEMPLATE = <<~MD
36
+ ## Prompt
37
+
38
+ Replace this with the prompt text (shown as typed input).
39
+
40
+ ## Reasoning
41
+
42
+ Optional -- delete this whole section if you don't want a "thinking"
43
+ trace. Replace this with the reasoning text; it streams first, then
44
+ is replaced by the response.
45
+
46
+ ## Output
47
+
48
+ Replace this with the response text (streamed as output). Code
49
+ fences work here too and are syntax-highlighted:
50
+
51
+ ```ruby
52
+ def hello
53
+ puts "hi"
54
+ end
55
+ ```
56
+ MD
57
+
58
+ # A starter Markdown file with the three recognized headings
59
+ # already in place, ready to fill in and pass to `-m`/`--markdown`.
60
+ def self.template
61
+ TEMPLATE
62
+ end
63
+
64
+ # Returns { prompt: "...", reasoning: "...", response: "..." },
65
+ # omitting keys whose section was absent from the file.
66
+ def self.parse(text)
67
+ sections = Hash.new { |h, k| h[k] = +"" }
68
+ current = nil
69
+ in_fence = false
70
+
71
+ text.each_line do |line|
72
+ stripped = line.chomp
73
+ in_fence = !in_fence if stripped =~ FENCE
74
+
75
+ if !in_fence && (m = HEADING.match(stripped)) && (key = SECTION_ALIASES[m[1].downcase])
76
+ current = key
77
+ next
78
+ end
79
+
80
+ sections[current] << line if current
81
+ end
82
+
83
+ sections.transform_values(&:strip)
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TokenReel
4
+ module Theme
5
+ PALETTES = {
6
+ dark: {
7
+ bg: "#0d1117", header: "#161b22", border: "#30363d",
8
+ fg: "#c9d1d9", prompt: "#7ee787", muted: "#8b949e",
9
+ dot_red: "#ff5f56", dot_yellow: "#ffbd2e", dot_green: "#27c93f",
10
+ syn_keyword: "#ff7b72", syn_string: "#a5d6ff", syn_number: "#79c0ff", syn_comment: "#8b949e"
11
+ },
12
+ matrix: {
13
+ bg: "#000000", header: "#000000", border: "#003300",
14
+ fg: "#00ff41", prompt: "#00ff41", muted: "#008f11",
15
+ dot_red: "#003300", dot_yellow: "#005500", dot_green: "#00ff41",
16
+ syn_keyword: "#00ffae", syn_string: "#7fff7f", syn_number: "#39ff14", syn_comment: "#008f11"
17
+ },
18
+ light: {
19
+ bg: "#ffffff", header: "#f0f0f0", border: "#d0d0d0",
20
+ fg: "#24292f", prompt: "#116329", muted: "#6e7781",
21
+ dot_red: "#ff5f56", dot_yellow: "#ffbd2e", dot_green: "#27c93f",
22
+ syn_keyword: "#cf222e", syn_string: "#0a3069", syn_number: "#0550ae", syn_comment: "#6e7781"
23
+ },
24
+ solarized: {
25
+ bg: "#002b36", header: "#073642", border: "#586e75",
26
+ fg: "#eee8d5", prompt: "#2aa198", muted: "#93a1a1",
27
+ dot_red: "#dc322f", dot_yellow: "#b58900", dot_green: "#859900",
28
+ syn_keyword: "#268bd2", syn_string: "#859900", syn_number: "#cb4b16", syn_comment: "#93a1a1"
29
+ }
30
+ }.freeze
31
+
32
+ def self.fetch(name)
33
+ PALETTES.fetch(name.to_sym) do
34
+ raise ConfigError, "unknown theme #{name.inspect} (valid: #{PALETTES.keys.join(', ')})"
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,120 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TokenReel
4
+ # A snapshot of the screen at a given moment: how much of the prompt,
5
+ # reasoning, and response are visible, which "phase" we're in, and
6
+ # whether the cursor happens to be in its "on" blink state.
7
+ State = Struct.new(:phase, :prompt_text, :reasoning_text, :response_text, :cursor_on, :dot_count, keyword_init: true) do
8
+ # Identical signatures render to the identical frame, so the
9
+ # sampler can skip re-rendering and just extend the previous
10
+ # frame's delay instead.
11
+ def signature
12
+ [phase, prompt_text, reasoning_text, response_text, cursor_on, dot_count]
13
+ end
14
+ end
15
+
16
+ # Pure function of time -> State. Doesn't touch the filesystem or
17
+ # ImageMagick at all, which makes it cheap to sample as densely as
18
+ # we like when building the frame list.
19
+ class Timeline
20
+ BLINK_HZ = 2.0 # cursor toggles this many times per second
21
+ DOT_INTERVAL = 0.35 # seconds between "thinking..." dot ticks
22
+
23
+ attr_reader :prompt_tokens, :reasoning_tokens, :response_tokens,
24
+ :prompt_full, :reasoning_full, :response_full,
25
+ :prompt_done_t, :thinking_end_t, :reasoning_end_t, :stream_end_t, :duration
26
+
27
+ def initialize(config)
28
+ @config = config
29
+ @prompt_tokens = Tokenizer.tokenize(config.prompt, config.unit)
30
+ @reasoning_tokens = Tokenizer.tokenize(config.reasoning, config.unit)
31
+ @response_tokens = Tokenizer.tokenize(config.response, config.unit)
32
+ @prompt_full = prompt_tokens.join
33
+ @reasoning_full = reasoning_tokens.join
34
+ @response_full = response_tokens.join
35
+
36
+ @prompt_interval = config.prompt_tps.to_f.positive? ? 1.0 / config.prompt_tps : 0
37
+ @reasoning_interval = config.reasoning_tps.to_f.positive? ? 1.0 / config.reasoning_tps : 1.0 / config.tps
38
+ @response_interval = 1.0 / config.tps
39
+
40
+ @prompt_done_t = @prompt_interval * prompt_tokens.size
41
+ @thinking_end_t = prompt_done_t + config.ttft.to_f
42
+ @reasoning_end_t = thinking_end_t + @reasoning_interval * reasoning_tokens.size
43
+ @stream_end_t = reasoning_end_t + @response_interval * response_tokens.size
44
+ @duration = stream_end_t + config.hold.to_f
45
+ end
46
+
47
+ def state_at(t)
48
+ t = t.clamp(0, duration)
49
+
50
+ if t < prompt_done_t
51
+ n = @prompt_interval.positive? ? (t / @prompt_interval).floor : prompt_tokens.size
52
+ State.new(
53
+ phase: :typing_prompt,
54
+ prompt_text: prompt_tokens[0...n].join,
55
+ reasoning_text: "",
56
+ response_text: "",
57
+ cursor_on: blink_on?(t),
58
+ dot_count: 0
59
+ )
60
+ elsif t < thinking_end_t
61
+ elapsed = t - prompt_done_t
62
+ State.new(
63
+ phase: :thinking,
64
+ prompt_text: prompt_full,
65
+ reasoning_text: "",
66
+ response_text: "",
67
+ cursor_on: blink_on?(t),
68
+ dot_count: ((elapsed / DOT_INTERVAL).to_i % 4)
69
+ )
70
+ elsif t < reasoning_end_t
71
+ elapsed = t - thinking_end_t
72
+ n = @reasoning_interval.positive? ? (elapsed / @reasoning_interval).floor : reasoning_tokens.size
73
+ n = n.clamp(0, reasoning_tokens.size)
74
+ State.new(
75
+ phase: :reasoning,
76
+ prompt_text: prompt_full,
77
+ reasoning_text: reasoning_tokens[0...n].join,
78
+ response_text: "",
79
+ cursor_on: blink_on?(t),
80
+ dot_count: 0
81
+ )
82
+ else
83
+ elapsed = t - reasoning_end_t
84
+ n = @response_interval.positive? ? (elapsed / @response_interval).floor : response_tokens.size
85
+ n = n.clamp(0, response_tokens.size)
86
+ done = n >= response_tokens.size
87
+ State.new(
88
+ phase: done ? :done : :streaming,
89
+ prompt_text: prompt_full,
90
+ reasoning_text: "",
91
+ response_text: response_tokens[0...n].join,
92
+ cursor_on: blink_on?(t),
93
+ dot_count: 0
94
+ )
95
+ end
96
+ end
97
+
98
+ # The fully-revealed, cursor-off state -- used to size the canvas
99
+ # so every frame in the GIF shares identical dimensions.
100
+ def final_state
101
+ State.new(phase: :done, prompt_text: prompt_full, reasoning_text: "", response_text: response_full,
102
+ cursor_on: false, dot_count: 0)
103
+ end
104
+
105
+ # The reasoning phase in full, cursor off -- used alongside
106
+ # final_state to size the canvas, since the reasoning trace (shown
107
+ # only while streaming, then replaced by the response) can be
108
+ # taller than the finished response.
109
+ def max_reasoning_state
110
+ State.new(phase: :reasoning, prompt_text: prompt_full, reasoning_text: reasoning_full, response_text: "",
111
+ cursor_on: false, dot_count: 0)
112
+ end
113
+
114
+ private
115
+
116
+ def blink_on?(t)
117
+ (t * (2 * BLINK_HZ)).to_i.even?
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TokenReel
4
+ # Splits text into the chunks that get revealed one-by-one while
5
+ # streaming. Joining the tokens back together always reproduces the
6
+ # original string exactly.
7
+ module Tokenizer
8
+ def self.tokenize(text, unit)
9
+ return [] if text.nil? || text.empty?
10
+
11
+ case unit.to_sym
12
+ when :char
13
+ text.each_char.to_a
14
+ when :word
15
+ # "word" + any trailing whitespace travels together, so a token
16
+ # reveals as a whole word (closer to how LLM tokens/BPE chunks
17
+ # tend to land) instead of one raw character at a time.
18
+ text.scan(/\S+\s*|\s+/)
19
+ else
20
+ raise ConfigError, "unknown unit #{unit.inspect} (valid: word, char)"
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TokenReel
4
+ VERSION = "0.2.0"
5
+ end
data/lib/token_reel.rb ADDED
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "shellwords"
4
+
5
+ require_relative "token_reel/version"
6
+ require_relative "token_reel/errors"
7
+ require_relative "token_reel/theme"
8
+ require_relative "token_reel/tokenizer"
9
+ require_relative "token_reel/script"
10
+ require_relative "token_reel/config"
11
+ require_relative "token_reel/timeline"
12
+ require_relative "token_reel/fonts"
13
+ require_relative "token_reel/highlight"
14
+ require_relative "token_reel/renderer"
15
+ require_relative "token_reel/gif_writer"
16
+ require_relative "token_reel/generator"
17
+ require_relative "token_reel/cli"
18
+
19
+ module TokenReel
20
+ # Convenience one-liner: TokenReel.generate(prompt: "...", response: "...", tps: 12)
21
+ def self.generate(**opts)
22
+ config = Config.new
23
+ opts.each do |k, v|
24
+ setter = "#{k}="
25
+ raise ConfigError, "unknown option #{k.inspect}" unless config.respond_to?(setter)
26
+
27
+ config.public_send(setter, v)
28
+ end
29
+ Generator.new(config).generate!
30
+ end
31
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "lib/token_reel/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "token_reel"
7
+ spec.version = TokenReel::VERSION
8
+ spec.authors = ["Thomas Powell"]
9
+ spec.email = ["twilliampowell@gmail.com"]
10
+ spec.summary = "Render terminal-style GIFs of an LLM prompt/response, streamed at a chosen tokens/sec and time-to-first-token"
11
+ spec.description = <<~DESC
12
+ token_reel renders a "CLI demo"-style animated GIF of a prompt being
13
+ answered: the prompt appears, there's a configurable pause (time to
14
+ first token), and the response streams in word-by-word or
15
+ character-by-character at a configurable tokens/sec rate -- handy for
16
+ READMEs, blog posts, and talks that want to show off an LLM CLI
17
+ without shelling out to a real model (or a real screen recorder).
18
+ DESC
19
+ spec.homepage = "https://github.com/stringsn88keys/token_reel"
20
+ spec.license = "MIT"
21
+ spec.required_ruby_version = ">= 3.0"
22
+
23
+ spec.metadata["homepage_uri"] = spec.homepage
24
+ spec.metadata["source_code_uri"] = spec.homepage
25
+
26
+ spec.files = Dir.chdir(__dir__) do
27
+ if system("git rev-parse --git-dir > /dev/null 2>&1")
28
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
29
+ else
30
+ Dir.glob("{lib,exe}/**/*", File::FNM_DOTMATCH).select { |f| File.file?(f) } +
31
+ %w[README.md token_reel.gemspec]
32
+ end
33
+ end
34
+ spec.bindir = "exe"
35
+ spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
36
+ spec.require_paths = ["lib"]
37
+
38
+ spec.add_development_dependency "rake", "~> 13.0"
39
+ spec.add_development_dependency "rspec", "~> 3.12"
40
+ end