token-lens 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +39 -7
- data/lib/token_lens/cli.rb +5 -3
- data/lib/token_lens/commands/record.rb +15 -2
- data/lib/token_lens/commands/render.rb +19 -4
- data/lib/token_lens/parser.rb +20 -1
- data/lib/token_lens/renderer/html.rb +435 -52
- data/lib/token_lens/renderer/layout.rb +18 -13
- data/lib/token_lens/renderer/reshaper.rb +27 -5
- data/lib/token_lens/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d15dc78e85a94ff7c3e4c22388e74a9084543cbe00936dcc26c155f2c1e07191
|
|
4
|
+
data.tar.gz: e66a0b7ba2753b5129f76c5900a8c81dcb6b51bdfb2a17b34e4a9d4c9925a7cc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9f5cdbfe0e769a8cac9c5d610336e66d1a8f9d07b0767bd31d3250fbe47425597eb89f5bc9e04e7a3ab8f97e2effe7c90d073f3fd8e02236ca4b210d6e81f6a3
|
|
7
|
+
data.tar.gz: 3fe4f1b44190dd712cf6c4353c3155fb0684a3192e987436a26611005a9775be23a09d0ddd34c2dc2845e678435ce0f2842068b335b245dadef994d17a3c009b
|
data/README.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Token Lens
|
|
2
2
|
|
|
3
|
+
[](https://rubygems.org/gems/token-lens)
|
|
4
|
+
[](https://rubygems.org/gems/token-lens)
|
|
5
|
+
[](https://github.com/Brickell-Research/token-lens/actions/workflows/ci.yml)
|
|
6
|
+
[](https://www.ruby-lang.org/)
|
|
7
|
+
[](https://github.com/standardrb/standard)
|
|
8
|
+
[](https://opensource.org/licenses/MIT)
|
|
9
|
+
|
|
3
10
|
Basically a combination of [perf](https://perfwiki.github.io/main/) plus [flame-graphs](https://www.brendangregg.com/flamegraphs.html) for local [claude-code](https://code.claude.com/docs/en/overview) usage.
|
|
4
11
|
|
|
5
12
|
## Architecture
|
|
@@ -16,19 +23,44 @@ Record Data --> Interpret Data --> Render Data
|
|
|
16
23
|
|
|
17
24
|
## Quick Start
|
|
18
25
|
|
|
26
|
+
### Zero setup — render any session right now
|
|
27
|
+
|
|
28
|
+
No recording needed. token-lens reads directly from Claude Code's session files:
|
|
19
29
|
|
|
20
|
-
**Locally**:
|
|
21
30
|
```
|
|
22
|
-
|
|
23
|
-
|
|
31
|
+
gem install token-lens
|
|
32
|
+
token-lens render
|
|
24
33
|
open flame.html
|
|
25
34
|
```
|
|
26
35
|
|
|
27
|
-
|
|
36
|
+
That's it. `render` with no arguments finds your most recent Claude Code session and renders it as a flame graph — no prior setup, no capture file, no extra terminal.
|
|
37
|
+
|
|
38
|
+
### Record a live session
|
|
39
|
+
|
|
40
|
+
If you want to capture a bounded window while you work (useful for comparing before/after):
|
|
41
|
+
|
|
28
42
|
```
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
token-lens render
|
|
43
|
+
token-lens record --duration-in-seconds=60
|
|
44
|
+
# ... do your Claude Code work in another terminal ...
|
|
45
|
+
token-lens render
|
|
46
|
+
open flame.html
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Captures auto-save to `~/.token-lens/sessions/<timestamp>.json`. `render` always picks the most recent capture first, then falls back to the live session JSONL if no captures exist.
|
|
50
|
+
|
|
51
|
+
### Options
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
token-lens record --duration-in-seconds=300 # record for 5 minutes
|
|
55
|
+
token-lens record --output=my-session.json # save to a specific path
|
|
56
|
+
token-lens render --file-path=my-session.json # render a specific capture
|
|
57
|
+
token-lens render --output=report.html # write HTML to a custom path
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Locally
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
bin/token-lens render
|
|
32
64
|
open flame.html
|
|
33
65
|
```
|
|
34
66
|
|
data/lib/token_lens/cli.rb
CHANGED
|
@@ -6,18 +6,20 @@ require "token_lens/commands/render"
|
|
|
6
6
|
|
|
7
7
|
module TokenLens
|
|
8
8
|
class CLI < Thor
|
|
9
|
-
desc "record", "Tail the active session and
|
|
9
|
+
desc "record", "Tail the active session and auto-save a capture file"
|
|
10
10
|
option :duration_in_seconds, type: :numeric, default: 30, desc: "Seconds to record"
|
|
11
11
|
option :project_dir, type: :string, desc: "Working directory of the Claude Code session to record (default: auto-detect)"
|
|
12
|
+
option :output, type: :string, desc: "Save path for the capture (default: ~/.token-lens/sessions/<timestamp>.json)"
|
|
12
13
|
def record
|
|
13
14
|
Commands::Record.new(
|
|
14
15
|
duration_in_seconds: options[:duration_in_seconds],
|
|
15
|
-
project_dir: options[:project_dir]
|
|
16
|
+
project_dir: options[:project_dir],
|
|
17
|
+
output: options[:output]
|
|
16
18
|
).run
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
desc "render", "Render a captured session as a flame graph"
|
|
20
|
-
option :file_path, type: :string,
|
|
22
|
+
option :file_path, type: :string, desc: "Path to the captured JSON file (default: most recent session)"
|
|
21
23
|
option :output, type: :string, default: "flame.html", desc: "Output HTML path"
|
|
22
24
|
def render
|
|
23
25
|
Commands::Render.new(file_path: options[:file_path], output: options[:output]).run
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
+
require "fileutils"
|
|
4
5
|
require "token_lens/sources/jsonl"
|
|
5
6
|
|
|
6
7
|
module TokenLens
|
|
7
8
|
module Commands
|
|
8
9
|
class Record
|
|
9
|
-
|
|
10
|
+
SESSIONS_DIR = Pathname.new(Dir.home).join(".token-lens", "sessions")
|
|
11
|
+
|
|
12
|
+
def initialize(duration_in_seconds:, project_dir: nil, output: nil)
|
|
10
13
|
@duration_in_seconds = duration_in_seconds
|
|
11
14
|
@project_dir = project_dir
|
|
15
|
+
@output = output
|
|
12
16
|
end
|
|
13
17
|
|
|
14
18
|
def run
|
|
@@ -34,9 +38,18 @@ module TokenLens
|
|
|
34
38
|
drain_thread.kill
|
|
35
39
|
events << queue.pop until queue.empty?
|
|
36
40
|
warn "\nCaptured #{events.size} events"
|
|
37
|
-
|
|
41
|
+
path = save_path
|
|
42
|
+
FileUtils.mkdir_p(path.dirname)
|
|
43
|
+
path.write(JSON.generate(events))
|
|
44
|
+
warn "Saved to #{path}"
|
|
38
45
|
exit 0
|
|
39
46
|
end
|
|
47
|
+
|
|
48
|
+
def save_path
|
|
49
|
+
return Pathname.new(@output) if @output
|
|
50
|
+
timestamp = Time.now.strftime("%Y-%m-%d_%H-%M-%S")
|
|
51
|
+
SESSIONS_DIR.join("#{timestamp}.json")
|
|
52
|
+
end
|
|
40
53
|
end
|
|
41
54
|
end
|
|
42
55
|
end
|
|
@@ -5,24 +5,39 @@ require "token_lens/renderer/reshaper"
|
|
|
5
5
|
require "token_lens/renderer/annotator"
|
|
6
6
|
require "token_lens/renderer/layout"
|
|
7
7
|
require "token_lens/renderer/html"
|
|
8
|
+
require "token_lens/session"
|
|
8
9
|
|
|
9
10
|
module TokenLens
|
|
10
11
|
module Commands
|
|
11
12
|
class Render
|
|
12
|
-
def initialize(
|
|
13
|
+
def initialize(output:, file_path: nil)
|
|
13
14
|
@file_path = file_path
|
|
14
15
|
@output = output
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
def run
|
|
18
|
-
|
|
19
|
+
path = resolve_path
|
|
20
|
+
warn "Rendering #{path}"
|
|
21
|
+
tree = Parser.new(file_path: path).parse
|
|
19
22
|
tree = Renderer::Reshaper.new.reshape(tree)
|
|
23
|
+
tree.sort_by! { |n| n[:token].timestamp || "" }
|
|
20
24
|
Renderer::Annotator.new.annotate(tree)
|
|
21
|
-
Renderer::Layout.new.layout(tree)
|
|
22
|
-
html = Renderer::Html.new.render(tree)
|
|
25
|
+
canvas_width = Renderer::Layout.new.layout(tree)
|
|
26
|
+
html = Renderer::Html.new(canvas_width: canvas_width).render(tree)
|
|
23
27
|
File.write(@output, html)
|
|
24
28
|
warn "Wrote #{@output}"
|
|
25
29
|
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def resolve_path
|
|
34
|
+
return @file_path if @file_path
|
|
35
|
+
sessions = Pathname.new(Dir.home).join(".token-lens", "sessions")
|
|
36
|
+
saved = sessions.glob("*.json").max_by(&:mtime)
|
|
37
|
+
return saved if saved
|
|
38
|
+
warn "No saved captures found — reading active Claude Code session directly"
|
|
39
|
+
Session.active_or_latest_jsonl
|
|
40
|
+
end
|
|
26
41
|
end
|
|
27
42
|
end
|
|
28
43
|
end
|
data/lib/token_lens/parser.rb
CHANGED
|
@@ -12,7 +12,7 @@ module TokenLens
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
def parse
|
|
15
|
-
raw_events =
|
|
15
|
+
raw_events = read_raw_events
|
|
16
16
|
tokens = raw_events
|
|
17
17
|
.map { |e| Tokens::Jsonl.from_raw(e) }
|
|
18
18
|
.select { |t| t.type == "user" || t.type == "assistant" }
|
|
@@ -162,6 +162,25 @@ module TokenLens
|
|
|
162
162
|
nodes.flat_map { |n| [n, *flatten_nodes(n[:children])] }
|
|
163
163
|
end
|
|
164
164
|
|
|
165
|
+
def read_raw_events
|
|
166
|
+
content = read_file
|
|
167
|
+
if content.lstrip.start_with?("[")
|
|
168
|
+
# Captured format: JSON array of {"event": {...}} wrappers produced by `record`
|
|
169
|
+
JSON.parse(content).map { |e| e["event"] }
|
|
170
|
+
else
|
|
171
|
+
# Raw JSONL: newline-delimited events from ~/.claude/projects/*/...jsonl
|
|
172
|
+
content.each_line.filter_map { |line|
|
|
173
|
+
begin
|
|
174
|
+
JSON.parse(line)
|
|
175
|
+
rescue
|
|
176
|
+
nil
|
|
177
|
+
end
|
|
178
|
+
}
|
|
179
|
+
end
|
|
180
|
+
rescue => e
|
|
181
|
+
raise TokenLens::ParseError, "Failed to parse #{@file_path}: #{e.message}"
|
|
182
|
+
end
|
|
183
|
+
|
|
165
184
|
def read_file
|
|
166
185
|
File.read(@file_path)
|
|
167
186
|
rescue => e
|
|
@@ -6,16 +6,21 @@ module TokenLens
|
|
|
6
6
|
module Renderer
|
|
7
7
|
class Html
|
|
8
8
|
ROW_HEIGHT = 32
|
|
9
|
-
COORD_WIDTH = 1200 # logical coordinate space
|
|
9
|
+
COORD_WIDTH = 1200 # default logical coordinate space
|
|
10
10
|
FONT_SIZE = 13
|
|
11
|
-
|
|
11
|
+
MIN_LABEL_PX = 60 # hide label if bar is narrower than 60px
|
|
12
|
+
|
|
13
|
+
def initialize(canvas_width: COORD_WIDTH)
|
|
14
|
+
@canvas_width = canvas_width
|
|
15
|
+
end
|
|
12
16
|
|
|
13
17
|
LEGEND_ITEMS = [
|
|
14
18
|
["bar-c-human", "User prompt"],
|
|
15
19
|
["bar-c-task", "Task callback"],
|
|
16
20
|
["bar-c-assistant", "Assistant response"],
|
|
17
21
|
["bar-c-tool", "Tool call"],
|
|
18
|
-
["bar-c-sidechain", "Subagent turn"]
|
|
22
|
+
["bar-c-sidechain", "Subagent turn"],
|
|
23
|
+
["bar-compaction", "Compaction"]
|
|
19
24
|
].freeze
|
|
20
25
|
|
|
21
26
|
CONTEXT_LIMIT = 200_000 # all current Claude models
|
|
@@ -28,7 +33,7 @@ module TokenLens
|
|
|
28
33
|
total_top = (max_depth + 1) * ROW_HEIGHT
|
|
29
34
|
total_tokens = nodes.sum { |n| n[:subtree_tokens] }
|
|
30
35
|
total_cost = nodes.sum { |n| n[:subtree_cost] }
|
|
31
|
-
total_tip = escape_js(escape_html(total_summary(
|
|
36
|
+
total_tip = escape_js(escape_html(total_summary(all_flat)))
|
|
32
37
|
@reread_files = build_reread_map(all)
|
|
33
38
|
@thread_count = nodes.length
|
|
34
39
|
@thread_numbers = {}
|
|
@@ -39,6 +44,8 @@ module TokenLens
|
|
|
39
44
|
next unless id && !@agent_labels.key?(id)
|
|
40
45
|
@agent_labels[id] = "A#{@agent_labels.size + 1}"
|
|
41
46
|
end
|
|
47
|
+
assign_alternation(nodes)
|
|
48
|
+
@hm_count = nodes.length # overridden by heatmap_html after grouping
|
|
42
49
|
token_total_lbl = "TOTAL · #{fmt(total_tokens)} tokens"
|
|
43
50
|
cost_total_lbl = "TOTAL · #{fmt_cost(total_cost)}"
|
|
44
51
|
|
|
@@ -56,23 +63,27 @@ module TokenLens
|
|
|
56
63
|
<body>
|
|
57
64
|
<div class="header">
|
|
58
65
|
<div>
|
|
59
|
-
<div class="summary">#{summary_text(
|
|
60
|
-
<div class="legend">#{legend_html}</div>
|
|
66
|
+
<div class="summary">#{summary_text(all_flat)}</div>
|
|
67
|
+
<div class="legend" id="legend" style="display:none">#{legend_html}</div>
|
|
61
68
|
</div>
|
|
62
69
|
<div class="header-btns">
|
|
63
70
|
<button class="theme-btn" id="theme-btn" onclick="toggleTheme()">◐ Light</button>
|
|
64
71
|
<button class="summary-btn" id="summary-btn" onclick="toggleSummary()">≡ Summary</button>
|
|
65
72
|
<button class="cost-btn" id="cost-btn" onclick="toggleCostView()">$ Cost view</button>
|
|
66
|
-
<button class="reset-btn" id="reset-btn" onclick="
|
|
73
|
+
<button class="reset-btn" id="reset-btn" onclick="resetZoom()">↩ Reset zoom</button>
|
|
74
|
+
<button class="export-btn" onclick="exportSVG()">⤓ Export</button>
|
|
67
75
|
</div>
|
|
68
76
|
</div>
|
|
69
|
-
|
|
70
|
-
<div class="
|
|
71
|
-
<div class="
|
|
77
|
+
#{heatmap_html(nodes)}
|
|
78
|
+
<div id="hm-back" class="hm-back" style="display:none" onclick="closePrompt()">← All prompts</div>
|
|
79
|
+
<div class="flame-wrap" id="flame-wrap" style="display:none">
|
|
80
|
+
<div class="flame" style="width:#{@canvas_width}px;height:#{flame_height}px">
|
|
81
|
+
<div class="bar total-bar" style="left:0%;width:100%;top:#{total_top}px" data-ox="0" data-ow="#{@canvas_width}" data-cx="0" data-cw="#{@canvas_width}" onmouseover="tip('#{total_tip}')" onmouseout="tip('')" onclick="if(hmActiveIdx<0)unzoom()"><span class="lbl total-lbl" id="total-lbl" data-token-text="#{token_total_lbl}" data-cost-text="#{cost_total_lbl}">#{token_total_lbl}</span></div>
|
|
72
82
|
#{all.map { |n| bar_html(n) }.join("\n")}
|
|
73
83
|
</div>
|
|
84
|
+
</div>
|
|
74
85
|
<div id="ftip" class="floattip"></div>
|
|
75
|
-
<div class="tip"
|
|
86
|
+
<div id="tip" class="tip"><span class="tip-label">Hover for details · Click to zoom</span></div>
|
|
76
87
|
#{session_summary_html(all_flat)}
|
|
77
88
|
<script>
|
|
78
89
|
#{js}
|
|
@@ -89,7 +100,7 @@ module TokenLens
|
|
|
89
100
|
end
|
|
90
101
|
|
|
91
102
|
def pct(val)
|
|
92
|
-
(val.to_f /
|
|
103
|
+
(val.to_f / @canvas_width * 100).round(4)
|
|
93
104
|
end
|
|
94
105
|
|
|
95
106
|
def bar_html(node)
|
|
@@ -99,10 +110,11 @@ module TokenLens
|
|
|
99
110
|
tip = escape_html(tooltip(node))
|
|
100
111
|
left = pct(node[:x])
|
|
101
112
|
width = pct(node[:w])
|
|
102
|
-
lbl_hidden = (
|
|
113
|
+
lbl_hidden = (node[:w] < MIN_LABEL_PX) ? " style=\"display:none\"" : ""
|
|
103
114
|
extra_class = " #{color_class(node)}"
|
|
115
|
+
extra_class += " bar-alt" if node[:alt]
|
|
104
116
|
extra_class += " bar-reread" if reread_bar?(node)
|
|
105
|
-
|
|
117
|
+
# is_compaction on assistant turns is for summary counting only; color is set via color_class on the user prompt
|
|
106
118
|
extra_class += " bar-pressure" if !t.is_compaction && context_pressure?(node)
|
|
107
119
|
ftip = escape_html(token_summary(node))
|
|
108
120
|
mouseover = if t.is_task_notification?
|
|
@@ -115,7 +127,7 @@ module TokenLens
|
|
|
115
127
|
end
|
|
116
128
|
badge = reread_bar?(node) ? "<span class=\"warn-badge\">\u26a0</span>" : ""
|
|
117
129
|
<<~HTML.chomp
|
|
118
|
-
<div class="bar#{extra_class}" style="left:#{left}%;width
|
|
130
|
+
<div class="bar#{extra_class}" style="left:#{left}%;width:calc(#{width}% + 1px);top:#{node[:y]}px" data-ox="#{node[:x]}" data-ow="#{node[:w]}" data-cx="#{node[:cost_x]}" data-cw="#{node[:cost_w]}" data-token-lbl="#{lbl}" data-cost-lbl="#{clbl}" data-ftip="#{ftip}" onmouseover="#{mouseover}" onmouseout="tip('')" onclick="zoom(this)"><span class="lbl"#{lbl_hidden}>#{lbl}</span>#{badge}</div>
|
|
119
131
|
HTML
|
|
120
132
|
end
|
|
121
133
|
|
|
@@ -184,11 +196,19 @@ module TokenLens
|
|
|
184
196
|
total_input = raw_input + cached + cache_new
|
|
185
197
|
hit_rate = (total_input > 0 && cached > 0) ? (cached.to_f / total_input * 100).round(0).to_i : nil
|
|
186
198
|
parts = []
|
|
199
|
+
timestamps = all.map { |n| n[:token].timestamp }.compact.sort
|
|
200
|
+
duration = begin
|
|
201
|
+
secs = (Time.parse(timestamps.last) - Time.parse(timestamps.first)).to_i
|
|
202
|
+
fmt_duration(secs) if timestamps.length >= 2 && secs > 0
|
|
203
|
+
rescue
|
|
204
|
+
nil
|
|
205
|
+
end
|
|
187
206
|
parts << "#{@thread_count} threads" if @thread_count&.> 1
|
|
188
207
|
parts << "#{prompts} #{"prompt".then { |w| (prompts == 1) ? w : "#{w}s" }}"
|
|
189
208
|
parts << "#{tasks} #{"task callback".then { |w| (tasks == 1) ? w : "#{w}s" }}" if tasks > 0
|
|
190
209
|
parts << "#{turns} main #{"turn".then { |w| (turns == 1) ? w : "#{w}s" }}"
|
|
191
210
|
parts << "#{sub} subagent #{"turn".then { |w| (sub == 1) ? w : "#{w}s" }}" if sub > 0
|
|
211
|
+
parts << duration if duration
|
|
192
212
|
parts << "fresh input: #{fmt(marginal)}" if marginal > 0
|
|
193
213
|
parts << "cached input: #{fmt(cached)}" if cached > 0
|
|
194
214
|
parts << "written to cache: #{fmt(cache_new)}" if cache_new > 0
|
|
@@ -214,6 +234,24 @@ module TokenLens
|
|
|
214
234
|
parts.join(" | ")
|
|
215
235
|
end
|
|
216
236
|
|
|
237
|
+
def thread_separators(all)
|
|
238
|
+
roots = all.select { |n| n[:depth] == 0 }
|
|
239
|
+
return "" if roots.size <= 1
|
|
240
|
+
# Emit a vertical line at the right edge of each thread (except the last)
|
|
241
|
+
roots[0..-2].map { |n|
|
|
242
|
+
right_x = n[:x] + n[:w]
|
|
243
|
+
pct_val = pct(right_x)
|
|
244
|
+
%(<div class="thread-sep" style="left:#{pct_val}%"></div>)
|
|
245
|
+
}.join("\n")
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def assign_alternation(siblings)
|
|
249
|
+
siblings.each_with_index do |node, i|
|
|
250
|
+
node[:alt] = i.odd?
|
|
251
|
+
assign_alternation(node[:children])
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
217
255
|
def legend_html
|
|
218
256
|
LEGEND_ITEMS.map { |css_class, lbl|
|
|
219
257
|
%(<span class="legend-item"><span class="legend-swatch #{css_class}"></span>#{lbl}</span>)
|
|
@@ -224,6 +262,7 @@ module TokenLens
|
|
|
224
262
|
t = node[:token]
|
|
225
263
|
return "bar-c-task" if t.is_task_notification?
|
|
226
264
|
return "bar-c-sidechain" if t.is_sidechain
|
|
265
|
+
return "bar-compaction" if t.is_human_prompt? && t.human_text.start_with?("This session is being continued")
|
|
227
266
|
return "bar-c-human" if t.is_human_prompt?
|
|
228
267
|
case t.role
|
|
229
268
|
when "user" then "bar-c-user"
|
|
@@ -350,7 +389,7 @@ module TokenLens
|
|
|
350
389
|
total_input = raw_input + cached + cache_new
|
|
351
390
|
hit_rate = (total_input > 0 && cached > 0) ? (cached.to_f / total_input * 100).round(1) : nil
|
|
352
391
|
total_cost = assistant_nodes.sum { |n| n[:token].cost_usd }
|
|
353
|
-
compactions =
|
|
392
|
+
compactions = all.count { |n| n[:token].is_human_prompt? && n[:token].human_text.start_with?("This session is being continued") }
|
|
354
393
|
pressure = assistant_nodes.count { |n| context_pressure?(n) }
|
|
355
394
|
models = assistant_nodes.map { |n| n[:token].model }.compact
|
|
356
395
|
.map { |m| model_short(m) }.uniq.join(", ")
|
|
@@ -432,6 +471,96 @@ module TokenLens
|
|
|
432
471
|
str.gsub("\\") { "\\\\" }.gsub("'") { "\\'" }.gsub("\n", "\\n").gsub("\r", "\\r")
|
|
433
472
|
end
|
|
434
473
|
|
|
474
|
+
def heatmap_color(value, min_val, max_val)
|
|
475
|
+
t = (max_val == min_val) ? 0.5 : ((value - min_val).to_f / (max_val - min_val))
|
|
476
|
+
t **= 0.7
|
|
477
|
+
r = (32 + (255 - 32) * t).round.clamp(0, 255)
|
|
478
|
+
g = (5 + (20 - 5) * t).round.clamp(0, 255)
|
|
479
|
+
b = (16 + (147 - 16) * t).round.clamp(0, 255)
|
|
480
|
+
"#%02x%02x%02x" % [r, g, b]
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
def heatmap_color_token(value, min_val, max_val)
|
|
484
|
+
t = (max_val == min_val) ? 0.5 : ((value - min_val).to_f / (max_val - min_val))
|
|
485
|
+
t **= 0.7
|
|
486
|
+
r = (7 + (0 - 7) * t).round.clamp(0, 255)
|
|
487
|
+
g = (48 + (206 - 48) * t).round.clamp(0, 255)
|
|
488
|
+
b = (48 + (209 - 48) * t).round.clamp(0, 255)
|
|
489
|
+
"#%02x%02x%02x" % [r, g, b]
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
def compaction_node?(node)
|
|
493
|
+
node[:token].is_human_prompt? && node[:token].human_text.start_with?("This session is being continued")
|
|
494
|
+
end
|
|
495
|
+
|
|
496
|
+
def heatmap_html(nodes)
|
|
497
|
+
# Merge each compaction node into the preceding group — it's overhead from that prompt
|
|
498
|
+
groups = []
|
|
499
|
+
nodes.each do |node|
|
|
500
|
+
if compaction_node?(node) && groups.any?
|
|
501
|
+
groups.last << node
|
|
502
|
+
else
|
|
503
|
+
groups << [node]
|
|
504
|
+
end
|
|
505
|
+
end
|
|
506
|
+
@hm_count = groups.length
|
|
507
|
+
|
|
508
|
+
group_tokens = groups.map { |g| g.sum { |n| n[:subtree_tokens] } }
|
|
509
|
+
group_costs = groups.map { |g| g.sum { |n| n[:subtree_cost] } }
|
|
510
|
+
min_tok, max_tok = group_tokens.min, group_tokens.max
|
|
511
|
+
min_cost, max_cost = group_costs.min, group_costs.max
|
|
512
|
+
|
|
513
|
+
cells = groups.each_with_index.map { |group, i|
|
|
514
|
+
primary = group.first
|
|
515
|
+
combined_tokens = group_tokens[i]
|
|
516
|
+
combined_cost = group_costs[i]
|
|
517
|
+
has_compaction = group.length > 1
|
|
518
|
+
|
|
519
|
+
color_cost = heatmap_color(combined_cost, min_cost, max_cost)
|
|
520
|
+
color_token = heatmap_color_token(combined_tokens, min_tok, max_tok)
|
|
521
|
+
|
|
522
|
+
# x/w spans all nodes in the group
|
|
523
|
+
ox = primary[:x]
|
|
524
|
+
ow = group.last[:x] + group.last[:w] - primary[:x]
|
|
525
|
+
cx = primary[:cost_x]
|
|
526
|
+
cw = group.last[:cost_x] + group.last[:cost_w] - primary[:cost_x]
|
|
527
|
+
|
|
528
|
+
num = @thread_numbers&.[](primary[:token].uuid)
|
|
529
|
+
tok_str = fmt(combined_tokens)
|
|
530
|
+
cost_str = (combined_cost > 0) ? " \u00b7 #{fmt_cost(combined_cost)}" : ""
|
|
531
|
+
compact_note = has_compaction ? " \u00b7 \u21ba compaction" : ""
|
|
532
|
+
base = num ? "Thread #{num}" : "Prompt #{i + 1}"
|
|
533
|
+
tip_text = escape_html(escape_js("#{base} \u00b7 #{tok_str} tokens#{cost_str}#{compact_note}"))
|
|
534
|
+
tip_prompt = escape_html(escape_js(primary[:token].human_text))
|
|
535
|
+
prompt_search = escape_html(truncate(primary[:token].human_text, 300).downcase)
|
|
536
|
+
|
|
537
|
+
badge = has_compaction ? %(<span class="hm-compact-badge">\u21ba</span>) : ""
|
|
538
|
+
%(<div class="hm-cell" tabindex="0" data-idx="#{i}" data-cost="#{combined_cost}" data-tokens="#{combined_tokens}" data-prompt="#{prompt_search}" data-color-cost="#{color_cost}" data-color-token="#{color_token}" data-ox="#{ox}" data-ow="#{ow}" data-cx="#{cx}" data-cw="#{cw}" data-tip="#{tip_text}" data-tip-prompt="#{tip_prompt}" style="background-color:#{color_token}" onmouseover="hmTip(this)" onmouseout="tip('')" onclick="openPrompt(#{i})"><span class="hm-idx">#{i + 1}</span>#{badge}</div>)
|
|
539
|
+
}.join("\n")
|
|
540
|
+
|
|
541
|
+
<<~HTML
|
|
542
|
+
<div id="heatmap" class="heatmap">
|
|
543
|
+
<div class="hm-meta">
|
|
544
|
+
<span class="hm-hint">Hover to preview · Click to open flame graph</span>
|
|
545
|
+
</div>
|
|
546
|
+
<div class="hm-controls">
|
|
547
|
+
<span class="hm-section-label">Prompts</span>
|
|
548
|
+
<input type="text" id="hm-search" class="hm-search" placeholder="Search prompts\u2026" oninput="filterHeatmap(this.value)">
|
|
549
|
+
<button class="hm-sort-btn" id="hm-sort-btn" onclick="sortHeatmap()">⇅ Sort by tokens</button>
|
|
550
|
+
<div class="hm-legend" id="hm-legend">
|
|
551
|
+
<span class="hm-legend-label" id="hm-legend-lo">few tokens</span>
|
|
552
|
+
<span class="hm-legend-ramp" id="hm-legend-ramp" data-token-grad="linear-gradient(to right, #073030, #00CED1)" data-cost-grad="linear-gradient(to right, #200510, #FF1493)" style="background:linear-gradient(to right, #073030, #00CED1)"></span>
|
|
553
|
+
<span class="hm-legend-label" id="hm-legend-hi">many</span>
|
|
554
|
+
</div>
|
|
555
|
+
</div>
|
|
556
|
+
<div class="hm-grid" id="hm-grid">
|
|
557
|
+
#{cells}
|
|
558
|
+
</div>
|
|
559
|
+
<div class="hm-empty" id="hm-empty">Click a prompt cell to explore its flame graph</div>
|
|
560
|
+
</div>
|
|
561
|
+
HTML
|
|
562
|
+
end
|
|
563
|
+
|
|
435
564
|
def css
|
|
436
565
|
<<~CSS
|
|
437
566
|
:root {
|
|
@@ -440,6 +569,7 @@ module TokenLens
|
|
|
440
569
|
--bar-assistant: #00CED1;
|
|
441
570
|
--bar-tool: #00A8E8;
|
|
442
571
|
--bar-sidechain: #C97BFF;
|
|
572
|
+
--bar-compaction: #607B8B;
|
|
443
573
|
--bar-user: #1a1a2e;
|
|
444
574
|
--bg: #000000;
|
|
445
575
|
--surface: #0a0a14;
|
|
@@ -466,7 +596,7 @@ module TokenLens
|
|
|
466
596
|
}
|
|
467
597
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
468
598
|
html, body { height: 100%; }
|
|
469
|
-
body { background: var(--bg); font-family: 'JetBrains Mono', 'Cascadia Mono', 'Fira Code', ui-monospace, monospace; font-size: #{FONT_SIZE}px; display: flex; flex-direction: column;
|
|
599
|
+
body { background: var(--bg); font-family: 'JetBrains Mono', 'Cascadia Mono', 'Fira Code', ui-monospace, monospace; font-size: #{FONT_SIZE}px; display: flex; flex-direction: column; height: 100%; color: var(--text); }
|
|
470
600
|
.header { display: flex; align-items: center; justify-content: space-between; padding: 4px 8px; border-bottom: 1px solid var(--border); }
|
|
471
601
|
.summary { color: var(--text-dim); font-size: 11px; line-height: 20px; }
|
|
472
602
|
.header-btns { display: flex; gap: 6px; align-items: center; }
|
|
@@ -477,33 +607,67 @@ module TokenLens
|
|
|
477
607
|
.reset-btn:hover { background: var(--surface); color: var(--text); border-color: var(--accent); }
|
|
478
608
|
.theme-btn { background: none; border: 1px solid var(--border); color: var(--text-dim); border-radius: 3px; padding: 2px 8px; font-size: 10px; cursor: pointer; font-family: inherit; }
|
|
479
609
|
.theme-btn:hover { background: var(--surface); color: var(--text); border-color: var(--accent2); }
|
|
480
|
-
.
|
|
481
|
-
.flame { position: relative;
|
|
610
|
+
.flame-wrap { flex: 1; display: flex; flex-direction: column; justify-content: flex-end; overflow-x: auto; overflow-y: hidden; width: 100%; }
|
|
611
|
+
.flame { position: relative; overflow: hidden; flex-shrink: 0; }
|
|
612
|
+
.heatmap { flex-shrink: 0; padding: 12px 8px 8px; background: var(--bg); border-bottom: 1px solid var(--border); }
|
|
613
|
+
.hm-meta { display: flex; align-items: center; justify-content: space-between; margin-bottom: 6px; }
|
|
614
|
+
.hm-controls { display: flex; gap: 8px; align-items: center; justify-content: center; margin-bottom: 8px; }
|
|
615
|
+
.hm-search { background: var(--surface); border: 1px solid var(--border); color: var(--text); border-radius: 3px; padding: 3px 8px; font-size: 11px; font-family: inherit; outline: none; width: 220px; }
|
|
616
|
+
.hm-search:focus { border-color: var(--accent2); }
|
|
617
|
+
.hm-sort-btn { background: none; border: 1px solid var(--border); color: var(--text-dim); border-radius: 3px; padding: 2px 8px; font-size: 10px; cursor: pointer; font-family: inherit; }
|
|
618
|
+
.hm-sort-btn:hover { background: var(--surface); color: var(--text); border-color: var(--accent2); }
|
|
619
|
+
.hm-sort-btn.active { border-color: var(--accent2); color: var(--accent2); }
|
|
620
|
+
.hm-grid { display: flex; flex-wrap: wrap; gap: 3px; justify-content: center; }
|
|
621
|
+
.hm-cell { width: 32px; height: 32px; border-radius: 3px; cursor: pointer; transition: opacity 0.1s ease-out, transform 0.1s ease-out, box-shadow 0.1s ease-out; flex-shrink: 0; box-shadow: inset 0 0 0 1px rgba(255,255,255,0.10); display: flex; align-items: center; justify-content: center; position: relative; }
|
|
622
|
+
.hm-cell:hover { transform: scale(1.08); z-index: 10; box-shadow: inset 0 0 0 1px rgba(255,255,255,0.10), 0 0 8px var(--accent-faint); }
|
|
623
|
+
.hm-cell:focus-visible { outline: 2px solid var(--accent2); outline-offset: 1px; z-index: 10; }
|
|
624
|
+
.hm-cell.hm-active { box-shadow: 0 0 0 2px var(--accent); transform: scale(1.15); z-index: 11; }
|
|
625
|
+
.hm-idx { font-size: 9px; font-weight: 700; color: rgba(255,255,255,0.6); pointer-events: none; line-height: 1; user-select: none; }
|
|
626
|
+
.hm-compact-badge { position: absolute; top: 2px; right: 3px; font-size: 8px; color: rgba(255,255,255,0.75); pointer-events: none; line-height: 1; }
|
|
627
|
+
.hm-cell.hm-dimmed { opacity: 0.15; }
|
|
628
|
+
.heatmap.hm-strip { padding: 4px 8px; border-bottom: 1px solid var(--border); }
|
|
629
|
+
.heatmap.hm-strip .hm-meta { display: none; }
|
|
630
|
+
.heatmap.hm-strip .hm-controls { display: none; }
|
|
631
|
+
.heatmap.hm-strip .hm-grid { gap: 2px; }
|
|
632
|
+
.heatmap.hm-strip .hm-cell { width: 16px; height: 16px; border-radius: 2px; }
|
|
633
|
+
.hm-back { padding: 4px 12px; font-size: 11px; color: var(--accent); cursor: pointer; border-bottom: 1px solid var(--border); background: var(--surface); user-select: none; }
|
|
634
|
+
.hm-back:hover { background: var(--border); }
|
|
635
|
+
.heatmap.hm-strip .hm-idx { display: none; }
|
|
636
|
+
.hm-section-label { color: var(--text-dim); font-size: 10px; font-weight: 600; letter-spacing: 0.06em; text-transform: uppercase; white-space: nowrap; }
|
|
637
|
+
.hm-hint { color: var(--text-dim); font-size: 10px; opacity: 0.6; white-space: nowrap; }
|
|
638
|
+
.hm-legend { display: flex; align-items: center; gap: 4px; }
|
|
639
|
+
.hm-legend-label { color: var(--text-dim); font-size: 9px; opacity: 0.7; white-space: nowrap; }
|
|
640
|
+
.hm-legend-ramp { width: 64px; height: 8px; border-radius: 2px; flex-shrink: 0; }
|
|
641
|
+
.hm-empty { color: var(--text-dim); font-size: 12px; text-align: center; padding: 40px 0 20px; opacity: 0.5; }
|
|
642
|
+
.heatmap.hm-strip .hm-empty { display: none; }
|
|
482
643
|
.bar {
|
|
483
644
|
position: absolute;
|
|
484
|
-
height: #{ROW_HEIGHT
|
|
485
|
-
border-radius:
|
|
486
|
-
border-right:
|
|
487
|
-
|
|
645
|
+
height: #{ROW_HEIGHT}px;
|
|
646
|
+
border-radius: 0;
|
|
647
|
+
border-right: 1.5px solid rgba(0,0,0,0.5);
|
|
648
|
+
box-shadow: inset 0 1px 0 rgba(255,255,255,0.10), inset 0 -1px 0 rgba(0,0,0,0.25);
|
|
488
649
|
cursor: pointer;
|
|
489
650
|
overflow: hidden;
|
|
490
651
|
user-select: none;
|
|
652
|
+
transition: left 0.2s ease, width 0.2s ease;
|
|
491
653
|
}
|
|
492
|
-
.bar:hover { filter: brightness(1.
|
|
654
|
+
.bar:hover { filter: brightness(1.18) saturate(1.1); }
|
|
493
655
|
.bar-reread { box-shadow: inset 0 -2px 0 var(--accent); }
|
|
494
|
-
.bar-compaction { box-shadow:
|
|
656
|
+
.bar-compaction { background: linear-gradient(to right, #7a96a8 0%, #607B8B 12%, #607B8B 88%, #4a6070 100%) !important; box-shadow: inset 0 1px 0 rgba(255,255,255,0.12), inset 0 -1px 0 rgba(0,0,0,0.3); }
|
|
495
657
|
.bar-pressure { box-shadow: 0 0 5px 1px rgba(255,20,147,0.4); }
|
|
658
|
+
.bar-alt { filter: brightness(0.82); }
|
|
496
659
|
.warn-badge { position: absolute; top: 1px; right: 3px; font-size: 10px; line-height: 1; color: var(--accent); pointer-events: none; font-weight: 600; }
|
|
497
|
-
.bar-c-human
|
|
498
|
-
.bar-c-task
|
|
499
|
-
.bar-c-assistant
|
|
500
|
-
.bar-c-tool
|
|
501
|
-
.bar-c-sidechain
|
|
502
|
-
.bar-c-user
|
|
660
|
+
.bar-c-human { background: linear-gradient(to right, #ff3faa 0%, #FF1493 12%, #FF1493 88%, #c8107a 100%); }
|
|
661
|
+
.bar-c-task { background: linear-gradient(to right, #ffc84e 0%, #FFB020 12%, #FFB020 88%, #d9941a 100%); }
|
|
662
|
+
.bar-c-assistant{ background: linear-gradient(to right, #1ee8eb 0%, #00CED1 12%, #00CED1 88%, #00abb0 100%); }
|
|
663
|
+
.bar-c-tool { background: linear-gradient(to right, #22c4ff 0%, #00A8E8 12%, #00A8E8 88%, #0088c0 100%); }
|
|
664
|
+
.bar-c-sidechain{ background: linear-gradient(to right, #d88fff 0%, #C97BFF 12%, #C97BFF 88%, #a85ee0 100%); }
|
|
665
|
+
.bar-c-user { background: var(--bar-user); }
|
|
666
|
+
.bar-c-sidechain .lbl, .bar-c-tool .lbl { color: rgba(255,255,255,0.9); }
|
|
503
667
|
.lbl {
|
|
504
668
|
display: block;
|
|
505
669
|
padding: 0 5px;
|
|
506
|
-
line-height: #{ROW_HEIGHT
|
|
670
|
+
line-height: #{ROW_HEIGHT}px;
|
|
507
671
|
overflow: hidden;
|
|
508
672
|
text-overflow: ellipsis;
|
|
509
673
|
white-space: nowrap;
|
|
@@ -525,10 +689,13 @@ module TokenLens
|
|
|
525
689
|
.legend { padding: 2px 8px 6px; display: flex; gap: 14px; flex-wrap: wrap; }
|
|
526
690
|
.legend-item { display: flex; align-items: center; gap: 5px; color: var(--text-dim); font-size: 10px; }
|
|
527
691
|
.legend-swatch { display: inline-block; width: 10px; height: 10px; border-radius: 1px; flex-shrink: 0; }
|
|
692
|
+
.legend-swatch.bar-compaction { background: var(--bar-compaction) !important; }
|
|
528
693
|
.floattip { position: fixed; background: var(--surface); color: var(--text); border: 1px solid var(--border); border-radius: 3px; padding: 3px 8px; font-size: 11px; font-family: inherit; pointer-events: none; display: none; white-space: nowrap; z-index: 1000; box-shadow: 0 4px 16px var(--shadow); }
|
|
529
694
|
.summary-btn { background: none; border: 1px solid var(--border); color: var(--text-dim); border-radius: 3px; padding: 2px 8px; font-size: 10px; cursor: pointer; font-family: inherit; }
|
|
530
695
|
.summary-btn:hover { background: var(--surface); color: var(--text); border-color: var(--accent); }
|
|
531
696
|
.summary-btn.active { border-color: var(--accent); color: var(--accent); }
|
|
697
|
+
.export-btn { background: none; border: 1px solid var(--border); color: var(--text-dim); border-radius: 3px; padding: 2px 8px; font-size: 10px; cursor: pointer; font-family: inherit; }
|
|
698
|
+
.export-btn:hover { background: var(--surface); color: var(--accent2); border-color: var(--accent2); }
|
|
532
699
|
.summary-panel { position: fixed; right: 0; top: 0; height: 100%; width: 260px; background: var(--surface); border-left: 1px solid var(--border); z-index: 500; overflow-y: auto; padding: 16px; box-shadow: -8px 0 24px var(--shadow); }
|
|
533
700
|
.summary-panel-title { color: var(--accent); font-size: 12px; font-weight: 600; margin-bottom: 14px; display: flex; justify-content: space-between; align-items: center; letter-spacing: 0.04em; }
|
|
534
701
|
.summary-close { background: none; border: none; color: var(--text-dim); font-size: 16px; cursor: pointer; padding: 0 2px; line-height: 1; font-family: inherit; }
|
|
@@ -541,12 +708,13 @@ module TokenLens
|
|
|
541
708
|
end
|
|
542
709
|
|
|
543
710
|
def js
|
|
544
|
-
w =
|
|
545
|
-
|
|
711
|
+
w = @canvas_width
|
|
712
|
+
min_lbl_px = MIN_LABEL_PX
|
|
546
713
|
<<~JS
|
|
547
714
|
(function() {
|
|
548
|
-
var W = #{w},
|
|
549
|
-
var costMode = false;
|
|
715
|
+
var W = #{w}, MIN_LBL_PX = #{min_lbl_px};
|
|
716
|
+
var costMode = false, zoomedEl = null;
|
|
717
|
+
var hmActiveIdx = -1, hmSorted = false, hmCount = #{@hm_count};
|
|
550
718
|
function bars() { return Array.from(document.querySelectorAll('.bar:not(.total-bar)')); }
|
|
551
719
|
function applyBar(el, nx, nw) {
|
|
552
720
|
if (nx + nw <= 0 || nx >= W) {
|
|
@@ -554,12 +722,19 @@ module TokenLens
|
|
|
554
722
|
} else {
|
|
555
723
|
el.style.display = '';
|
|
556
724
|
el.style.left = (nx / W * 100) + '%';
|
|
557
|
-
el.style.width = Math.max(nw, 1) / W * 100 + '%';
|
|
725
|
+
el.style.width = 'calc(' + (Math.max(nw, 1) / W * 100) + '% + 1px)';
|
|
558
726
|
var lbl = el.querySelector('.lbl');
|
|
559
|
-
if (lbl) lbl.style.display = (nw
|
|
727
|
+
if (lbl) lbl.style.display = (nw < MIN_LBL_PX) ? 'none' : '';
|
|
560
728
|
}
|
|
561
729
|
}
|
|
562
730
|
function resetBtn() { return document.getElementById('reset-btn'); }
|
|
731
|
+
function withoutTransition(fn) {
|
|
732
|
+
document.querySelectorAll('.bar').forEach(function(b) { b.style.transition = 'none'; });
|
|
733
|
+
fn();
|
|
734
|
+
var first = document.querySelector('.bar');
|
|
735
|
+
if (first) first.offsetHeight;
|
|
736
|
+
document.querySelectorAll('.bar').forEach(function(b) { b.style.transition = ''; });
|
|
737
|
+
}
|
|
563
738
|
window.toggleTheme = function() {
|
|
564
739
|
var root = document.documentElement;
|
|
565
740
|
var isLight = root.getAttribute('data-theme') === 'light';
|
|
@@ -576,29 +751,52 @@ module TokenLens
|
|
|
576
751
|
if (btn) btn.classList.toggle('active', !shown);
|
|
577
752
|
};
|
|
578
753
|
window.toggleCostView = function() {
|
|
754
|
+
var wasZoomed = zoomedEl;
|
|
755
|
+
if (wasZoomed) unzoom();
|
|
579
756
|
costMode = !costMode;
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
}
|
|
757
|
+
withoutTransition(function() {
|
|
758
|
+
bars().forEach(function(b) {
|
|
759
|
+
var nx = costMode ? +b.getAttribute('data-cx') : +b.getAttribute('data-ox');
|
|
760
|
+
var nw = costMode ? +b.getAttribute('data-cw') : +b.getAttribute('data-ow');
|
|
761
|
+
applyBar(b, nx, nw);
|
|
762
|
+
var lbl = b.querySelector('.lbl');
|
|
763
|
+
if (lbl) {
|
|
764
|
+
var text = b.getAttribute(costMode ? 'data-cost-lbl' : 'data-token-lbl');
|
|
765
|
+
if (text !== null) lbl.textContent = text;
|
|
766
|
+
}
|
|
767
|
+
});
|
|
591
768
|
});
|
|
592
769
|
var tl = document.getElementById('total-lbl');
|
|
593
770
|
if (tl) tl.innerHTML = tl.getAttribute(costMode ? 'data-cost-text' : 'data-token-text');
|
|
594
771
|
var cb = document.getElementById('cost-btn');
|
|
595
772
|
if (cb) { cb.textContent = costMode ? '# Token view' : '$ Cost view'; cb.classList.toggle('active', costMode); }
|
|
596
|
-
|
|
773
|
+
hmCells().forEach(function(c) { c.style.backgroundColor = c.getAttribute(costMode ? 'data-color-cost' : 'data-color-token'); });
|
|
774
|
+
var sb = document.getElementById('hm-sort-btn');
|
|
775
|
+
if (sb && hmSorted) sb.innerHTML = costMode ? '⇅ Sort by cost' : '⇅ Sort by tokens';
|
|
776
|
+
var ramp = document.getElementById('hm-legend-ramp');
|
|
777
|
+
if (ramp) ramp.style.background = costMode ? ramp.getAttribute('data-cost-grad') : ramp.getAttribute('data-token-grad');
|
|
778
|
+
var lmLo = document.getElementById('hm-legend-lo');
|
|
779
|
+
var lmHi = document.getElementById('hm-legend-hi');
|
|
780
|
+
if (lmLo) lmLo.textContent = costMode ? 'cheap' : 'few tokens';
|
|
781
|
+
if (lmHi) lmHi.textContent = costMode ? 'costly' : 'many';
|
|
782
|
+
scaleHeatmap();
|
|
783
|
+
if (wasZoomed) {
|
|
784
|
+
if (wasZoomed.classList && wasZoomed.classList.contains('hm-cell')) {
|
|
785
|
+
openPrompt(+wasZoomed.getAttribute('data-idx'));
|
|
786
|
+
} else {
|
|
787
|
+
zoom(wasZoomed);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
597
790
|
};
|
|
598
791
|
window.zoom = function(el) {
|
|
599
792
|
var fx = costMode ? +el.getAttribute('data-cx') : +el.getAttribute('data-ox');
|
|
600
793
|
var fw = costMode ? +el.getAttribute('data-cw') : +el.getAttribute('data-ow');
|
|
601
794
|
if (fw >= W - 1) { unzoom(); return; }
|
|
795
|
+
var wrap = document.getElementById('flame-wrap');
|
|
796
|
+
var flame = document.querySelector('.flame');
|
|
797
|
+
if (!flame.getAttribute('data-orig-w')) flame.setAttribute('data-orig-w', flame.style.width);
|
|
798
|
+
flame.style.width = wrap.clientWidth + 'px';
|
|
799
|
+
wrap.scrollLeft = 0;
|
|
602
800
|
bars().forEach(function(b) {
|
|
603
801
|
if (!b.getAttribute('ox')) {
|
|
604
802
|
b.setAttribute('ox', costMode ? +b.getAttribute('data-cx') : +b.getAttribute('data-ox'));
|
|
@@ -607,8 +805,12 @@ module TokenLens
|
|
|
607
805
|
applyBar(b, (+b.getAttribute('ox') - fx) / fw * W, +b.getAttribute('ow') / fw * W);
|
|
608
806
|
});
|
|
609
807
|
var btn = resetBtn(); if (btn) btn.style.display = 'inline-block';
|
|
808
|
+
zoomedEl = el;
|
|
610
809
|
};
|
|
611
810
|
window.unzoom = function() {
|
|
811
|
+
var flame = document.querySelector('.flame');
|
|
812
|
+
var origW = flame.getAttribute('data-orig-w');
|
|
813
|
+
if (origW) { flame.style.width = origW; flame.removeAttribute('data-orig-w'); }
|
|
612
814
|
bars().forEach(function(b) {
|
|
613
815
|
var ox = b.getAttribute('ox');
|
|
614
816
|
if (ox) {
|
|
@@ -618,12 +820,65 @@ module TokenLens
|
|
|
618
820
|
}
|
|
619
821
|
});
|
|
620
822
|
var btn = resetBtn(); if (btn) btn.style.display = 'none';
|
|
823
|
+
zoomedEl = null;
|
|
824
|
+
};
|
|
825
|
+
// resetZoom: if sub-zoomed inside a prompt → unzoom back to prompt view;
|
|
826
|
+
// if viewing a prompt (or full overview) → close back to heatmap
|
|
827
|
+
window.resetZoom = function() {
|
|
828
|
+
if (zoomedEl && zoomedEl.classList && !zoomedEl.classList.contains('hm-cell')) {
|
|
829
|
+
unzoom();
|
|
830
|
+
} else {
|
|
831
|
+
closePrompt();
|
|
832
|
+
}
|
|
833
|
+
};
|
|
834
|
+
function escSVG(s) { return s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"'); }
|
|
835
|
+
window.exportSVG = function() {
|
|
836
|
+
var SVG_W = 1600, HDR = 28, MIN_H = 400;
|
|
837
|
+
var flame = document.querySelector('.flame');
|
|
838
|
+
var flameH = parseFloat(flame.style.height) || flame.offsetHeight;
|
|
839
|
+
var svgH = Math.max(MIN_H, Math.round(flameH) + HDR);
|
|
840
|
+
var yOffset = svgH - Math.round(flameH) - HDR; // push bars to bottom
|
|
841
|
+
var allBars = Array.from(document.querySelectorAll('.flame .bar')).filter(function(b) { return b.style.display !== 'none'; });
|
|
842
|
+
if (!allBars.length) return;
|
|
843
|
+
var st = getComputedStyle(document.documentElement);
|
|
844
|
+
var bg = st.getPropertyValue('--bg').trim() || '#000';
|
|
845
|
+
var surface = st.getPropertyValue('--surface').trim() || '#0a0a14';
|
|
846
|
+
var border = st.getPropertyValue('--border').trim() || '#1a1a2e';
|
|
847
|
+
var textDim = st.getPropertyValue('--text-dim').trim() || '#8892a0';
|
|
848
|
+
var barText = st.getPropertyValue('--bar-text').trim() || '#000';
|
|
849
|
+
var summaryEl = document.querySelector('.summary');
|
|
850
|
+
var summaryStr = summaryEl ? summaryEl.textContent.trim() : '';
|
|
851
|
+
var defs = [], rects = [], texts = [];
|
|
852
|
+
rects.push('<rect x="0" y="0" width="' + SVG_W + '" height="' + HDR + '" fill="' + surface + '"/>');
|
|
853
|
+
rects.push('<line x1="0" y1="' + HDR + '" x2="' + SVG_W + '" y2="' + HDR + '" stroke="' + border + '" stroke-width="1"/>');
|
|
854
|
+
texts.push('<text x="8" y="18" font-family="monospace" font-size="11" fill="' + textDim + '">' + escSVG(summaryStr) + '</text>');
|
|
855
|
+
allBars.forEach(function(bar, i) {
|
|
856
|
+
var x = Math.round(parseFloat(bar.style.left) / 100 * SVG_W);
|
|
857
|
+
var w = Math.max(1, Math.round(parseFloat(bar.style.width) / 100 * SVG_W));
|
|
858
|
+
var y = Math.round(parseFloat(bar.style.top)) + HDR + yOffset;
|
|
859
|
+
var h = Math.round(parseFloat(getComputedStyle(bar).height));
|
|
860
|
+
if (isNaN(x) || isNaN(y) || isNaN(w) || h <= 0) return;
|
|
861
|
+
var fill = getComputedStyle(bar).backgroundColor;
|
|
862
|
+
var cid = 'c' + i;
|
|
863
|
+
defs.push('<clipPath id="' + cid + '"><rect x="' + x + '" y="' + y + '" width="' + w + '" height="' + h + '"/></clipPath>');
|
|
864
|
+
rects.push('<rect x="' + x + '" y="' + y + '" width="' + w + '" height="' + h + '" fill="' + fill + '"/>');
|
|
865
|
+
var lbl = bar.querySelector('.lbl');
|
|
866
|
+
if (lbl && lbl.style.display !== 'none' && w > 28 && lbl.textContent.trim()) {
|
|
867
|
+
var isTotal = bar.classList.contains('total-bar');
|
|
868
|
+
texts.push('<text x="' + (x+3) + '" y="' + (y + h/2 + 4) + '" font-family="monospace" font-size="11" fill="' + (isTotal ? textDim : barText) + '" font-weight="' + (isTotal ? '600' : '400') + '" clip-path="url(#' + cid + ')">' + escSVG(lbl.textContent) + '</text>');
|
|
869
|
+
}
|
|
870
|
+
});
|
|
871
|
+
var svg = '<?xml version="1.0" encoding="UTF-8"?><svg xmlns="http://www.w3.org/2000/svg" width="' + SVG_W + '" height="' + svgH + '"><rect width="100%" height="100%" fill="' + bg + '"/><defs>' + defs.join('') + '</defs>' + rects.join('') + texts.join('') + '</svg>';
|
|
872
|
+
var a = document.createElement('a');
|
|
873
|
+
a.href = URL.createObjectURL(new Blob([svg], {type: 'image/svg+xml'}));
|
|
874
|
+
a.download = 'token-lens.svg';
|
|
875
|
+
document.body.appendChild(a); a.click(); document.body.removeChild(a);
|
|
621
876
|
};
|
|
622
877
|
function esc(t) { return t.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>'); }
|
|
623
878
|
window.tip = function(s, prompt) {
|
|
624
879
|
var el = document.getElementById('tip');
|
|
625
880
|
if (!el) return;
|
|
626
|
-
if (!s && !prompt) { el.innerHTML = '
|
|
881
|
+
if (!s && !prompt) { el.innerHTML = '<span class="tip-label">Hover for details \u00b7 Click to zoom</span>'; return; }
|
|
627
882
|
var sep = '<span class="tip-sep">\u00b7</span>';
|
|
628
883
|
var parts = s ? s.split(' | ').filter(Boolean) : [];
|
|
629
884
|
var html = parts.map(function(p, i) {
|
|
@@ -634,7 +889,7 @@ module TokenLens
|
|
|
634
889
|
if (/^[^:]*:\s/.test(p)) return '<span class="tip-label">' + e + '</span>';
|
|
635
890
|
return '<span class="tip-code">' + e + '</span>';
|
|
636
891
|
}).join(sep);
|
|
637
|
-
if (prompt) html += '<span class="tip-prompt">' + esc(prompt) + '</span>';
|
|
892
|
+
if (prompt) html += (html ? sep : '') + '<span class="tip-prompt">' + esc(prompt) + '</span>';
|
|
638
893
|
el.innerHTML = html;
|
|
639
894
|
};
|
|
640
895
|
var mx = 0, my = 0;
|
|
@@ -659,6 +914,134 @@ module TokenLens
|
|
|
659
914
|
var bar = e.target.closest && e.target.closest('.bar:not(.total-bar)');
|
|
660
915
|
if (bar) { var ft = document.getElementById('ftip'); if (ft) ft.style.display = 'none'; }
|
|
661
916
|
});
|
|
917
|
+
// Disable transitions on initial load
|
|
918
|
+
document.querySelectorAll('.bar').forEach(function(b) { b.style.transition = 'none'; });
|
|
919
|
+
requestAnimationFrame(function() { requestAnimationFrame(function() {
|
|
920
|
+
document.querySelectorAll('.bar').forEach(function(b) { b.style.transition = ''; });
|
|
921
|
+
}); });
|
|
922
|
+
// Heatmap
|
|
923
|
+
function hmCells() { return Array.from(document.querySelectorAll('.hm-cell')); }
|
|
924
|
+
window.hmTip = function(el) { tip(el.getAttribute('data-tip'), el.getAttribute('data-tip-prompt')); };
|
|
925
|
+
function zoomToRoot(cell) {
|
|
926
|
+
var fx = costMode ? +cell.getAttribute('data-cx') : +cell.getAttribute('data-ox');
|
|
927
|
+
var fw = costMode ? +cell.getAttribute('data-cw') : +cell.getAttribute('data-ow');
|
|
928
|
+
if (fw <= 0) return;
|
|
929
|
+
var wrap = document.getElementById('flame-wrap');
|
|
930
|
+
var flame = document.querySelector('.flame');
|
|
931
|
+
if (!flame.getAttribute('data-orig-w')) flame.setAttribute('data-orig-w', flame.style.width);
|
|
932
|
+
flame.style.width = wrap.clientWidth + 'px';
|
|
933
|
+
wrap.scrollLeft = 0;
|
|
934
|
+
bars().forEach(function(b) {
|
|
935
|
+
if (!b.getAttribute('ox')) {
|
|
936
|
+
b.setAttribute('ox', costMode ? +b.getAttribute('data-cx') : +b.getAttribute('data-ox'));
|
|
937
|
+
b.setAttribute('ow', costMode ? +b.getAttribute('data-cw') : +b.getAttribute('data-ow'));
|
|
938
|
+
}
|
|
939
|
+
applyBar(b, (+b.getAttribute('ox') - fx) / fw * W, +b.getAttribute('ow') / fw * W);
|
|
940
|
+
});
|
|
941
|
+
zoomedEl = cell;
|
|
942
|
+
}
|
|
943
|
+
window.openPrompt = function(idx) {
|
|
944
|
+
hmActiveIdx = idx;
|
|
945
|
+
var hm = document.getElementById('heatmap');
|
|
946
|
+
if (hm) hm.classList.add('hm-strip');
|
|
947
|
+
hmCells().forEach(function(c) { c.classList.toggle('hm-active', +c.getAttribute('data-idx') === idx); });
|
|
948
|
+
var fw = document.getElementById('flame-wrap');
|
|
949
|
+
if (fw) fw.style.display = '';
|
|
950
|
+
var lg = document.getElementById('legend'); if (lg) lg.style.display = '';
|
|
951
|
+
var back = document.getElementById('hm-back');
|
|
952
|
+
if (back) back.style.display = '';
|
|
953
|
+
var cell = document.querySelector('.hm-cell[data-idx="' + idx + '"]');
|
|
954
|
+
if (cell) { withoutTransition(function() { zoomToRoot(cell); }); }
|
|
955
|
+
var btn = resetBtn(); if (btn) btn.style.display = 'none';
|
|
956
|
+
};
|
|
957
|
+
window.closePrompt = function() {
|
|
958
|
+
hmActiveIdx = -1;
|
|
959
|
+
unzoom();
|
|
960
|
+
var fw = document.getElementById('flame-wrap');
|
|
961
|
+
if (fw) fw.style.display = 'none';
|
|
962
|
+
var lg = document.getElementById('legend'); if (lg) lg.style.display = 'none';
|
|
963
|
+
var hm = document.getElementById('heatmap');
|
|
964
|
+
if (hm) hm.classList.remove('hm-strip');
|
|
965
|
+
hmCells().forEach(function(c) { c.classList.remove('hm-active'); });
|
|
966
|
+
var back = document.getElementById('hm-back');
|
|
967
|
+
if (back) back.style.display = 'none';
|
|
968
|
+
var si = document.getElementById('hm-search');
|
|
969
|
+
if (si) { si.value = ''; filterHeatmap(''); }
|
|
970
|
+
};
|
|
971
|
+
window.sortHeatmap = function() {
|
|
972
|
+
hmSorted = !hmSorted;
|
|
973
|
+
var grid = document.getElementById('hm-grid');
|
|
974
|
+
if (!grid) return;
|
|
975
|
+
var cells = hmCells();
|
|
976
|
+
if (hmSorted) {
|
|
977
|
+
cells.sort(function(a, b) {
|
|
978
|
+
var av = costMode ? parseFloat(a.getAttribute('data-cost')) : parseInt(a.getAttribute('data-tokens'));
|
|
979
|
+
var bv = costMode ? parseFloat(b.getAttribute('data-cost')) : parseInt(b.getAttribute('data-tokens'));
|
|
980
|
+
return bv - av;
|
|
981
|
+
});
|
|
982
|
+
} else {
|
|
983
|
+
cells.sort(function(a, b) { return parseInt(a.getAttribute('data-idx')) - parseInt(b.getAttribute('data-idx')); });
|
|
984
|
+
}
|
|
985
|
+
cells.forEach(function(c) { grid.appendChild(c); });
|
|
986
|
+
scaleHeatmap();
|
|
987
|
+
var btn = document.getElementById('hm-sort-btn');
|
|
988
|
+
if (btn) btn.classList.toggle('active', hmSorted);
|
|
989
|
+
};
|
|
990
|
+
window.filterHeatmap = function(query) {
|
|
991
|
+
var q = query.toLowerCase().trim();
|
|
992
|
+
hmCells().forEach(function(c) {
|
|
993
|
+
var match = !q || (c.getAttribute('data-prompt') || '').indexOf(q) !== -1;
|
|
994
|
+
c.classList.toggle('hm-dimmed', !match);
|
|
995
|
+
});
|
|
996
|
+
scaleHeatmap();
|
|
997
|
+
};
|
|
998
|
+
function scaleHeatmap() {
|
|
999
|
+
var hm = document.getElementById('heatmap');
|
|
1000
|
+
if (hm && hm.classList.contains('hm-strip')) return;
|
|
1001
|
+
var allCells = hmCells();
|
|
1002
|
+
var cells = allCells.filter(function(c) { return !c.classList.contains('hm-dimmed'); });
|
|
1003
|
+
var n = cells.length || 1;
|
|
1004
|
+
var maxCell = Math.min(96, Math.max(36, Math.floor(1400 / Math.sqrt(n))));
|
|
1005
|
+
var minCell = 16;
|
|
1006
|
+
var vals = cells.map(function(c) {
|
|
1007
|
+
return costMode ? parseFloat(c.getAttribute('data-cost')) : parseInt(c.getAttribute('data-tokens'));
|
|
1008
|
+
});
|
|
1009
|
+
var maxVal = Math.max.apply(null, vals) || 1;
|
|
1010
|
+
var minVal = Math.min.apply(null, vals) || 0;
|
|
1011
|
+
cells.forEach(function(c, i) {
|
|
1012
|
+
var v = vals[i];
|
|
1013
|
+
var t = (maxVal === minVal) ? 0.5 : (v - minVal) / (maxVal - minVal);
|
|
1014
|
+
var sz = Math.round(minCell + (maxCell - minCell) * Math.sqrt(t));
|
|
1015
|
+
c.style.width = sz + 'px';
|
|
1016
|
+
c.style.height = sz + 'px';
|
|
1017
|
+
var lbl = c.querySelector('.hm-idx');
|
|
1018
|
+
if (lbl) lbl.style.display = (sz < 24) ? 'none' : '';
|
|
1019
|
+
});
|
|
1020
|
+
allCells.filter(function(c) { return c.classList.contains('hm-dimmed'); }).forEach(function(c) {
|
|
1021
|
+
c.style.width = minCell + 'px';
|
|
1022
|
+
c.style.height = minCell + 'px';
|
|
1023
|
+
});
|
|
1024
|
+
}
|
|
1025
|
+
document.addEventListener('keydown', function(e) {
|
|
1026
|
+
if (e.key === 'Enter' && document.activeElement && document.activeElement.classList.contains('hm-cell')) {
|
|
1027
|
+
e.preventDefault();
|
|
1028
|
+
openPrompt(+document.activeElement.getAttribute('data-idx'));
|
|
1029
|
+
return;
|
|
1030
|
+
}
|
|
1031
|
+
if (hmActiveIdx < 0) return;
|
|
1032
|
+
if (e.target && e.target.tagName === 'INPUT') return;
|
|
1033
|
+
if (e.key === 'ArrowLeft' || e.key === 'ArrowUp') {
|
|
1034
|
+
e.preventDefault();
|
|
1035
|
+
if (hmActiveIdx > 0) openPrompt(hmActiveIdx - 1);
|
|
1036
|
+
} else if (e.key === 'ArrowRight' || e.key === 'ArrowDown') {
|
|
1037
|
+
e.preventDefault();
|
|
1038
|
+
if (hmActiveIdx < hmCount - 1) openPrompt(hmActiveIdx + 1);
|
|
1039
|
+
} else if (e.key === 'Escape') {
|
|
1040
|
+
e.preventDefault();
|
|
1041
|
+
closePrompt();
|
|
1042
|
+
}
|
|
1043
|
+
});
|
|
1044
|
+
scaleHeatmap();
|
|
662
1045
|
})();
|
|
663
1046
|
JS
|
|
664
1047
|
end
|
|
@@ -5,6 +5,7 @@ module TokenLens
|
|
|
5
5
|
class Layout
|
|
6
6
|
CANVAS_WIDTH = 1200
|
|
7
7
|
ROW_HEIGHT = 32
|
|
8
|
+
MIN_THREAD_WIDTH = 80 # minimum pixels per root thread so slivers are readable
|
|
8
9
|
|
|
9
10
|
def initialize(canvas_width: CANVAS_WIDTH)
|
|
10
11
|
@canvas_width = canvas_width
|
|
@@ -12,15 +13,17 @@ module TokenLens
|
|
|
12
13
|
|
|
13
14
|
def layout(nodes)
|
|
14
15
|
max_depth = all_nodes(nodes).map { |n| n[:depth] }.max || 0
|
|
16
|
+
effective_width = [@canvas_width, nodes.length * MIN_THREAD_WIDTH].max
|
|
17
|
+
|
|
15
18
|
total = nodes.sum { |n| n[:subtree_tokens] }
|
|
16
|
-
scale = (total > 0) ?
|
|
19
|
+
scale = (total > 0) ? effective_width.to_f / total : 1.0
|
|
17
20
|
position(nodes, x: 0, scale: scale, max_depth: max_depth)
|
|
18
21
|
|
|
19
22
|
total_cost = nodes.sum { |n| n[:subtree_cost] }
|
|
20
|
-
cost_scale = (total_cost > 0) ?
|
|
23
|
+
cost_scale = (total_cost > 0) ? effective_width.to_f / total_cost : 1.0
|
|
21
24
|
position_cost(nodes, x: 0, scale: cost_scale, max_depth: max_depth)
|
|
22
25
|
|
|
23
|
-
|
|
26
|
+
effective_width
|
|
24
27
|
end
|
|
25
28
|
|
|
26
29
|
private
|
|
@@ -28,23 +31,25 @@ module TokenLens
|
|
|
28
31
|
# Bottom-up layout: roots at bottom (y = max_depth * ROW_HEIGHT),
|
|
29
32
|
# deepest children at top (y = 0).
|
|
30
33
|
def position(nodes, x:, scale:, max_depth:)
|
|
31
|
-
cursor = x
|
|
34
|
+
cursor = x.to_f
|
|
32
35
|
nodes.each do |node|
|
|
33
|
-
|
|
36
|
+
start = cursor.round
|
|
37
|
+
cursor += node[:subtree_tokens] * scale
|
|
38
|
+
node[:x] = start
|
|
34
39
|
node[:y] = (max_depth - node[:depth]) * ROW_HEIGHT
|
|
35
|
-
node[:w] =
|
|
36
|
-
position(node[:children], x:
|
|
37
|
-
cursor += node[:w]
|
|
40
|
+
node[:w] = cursor.round - start
|
|
41
|
+
position(node[:children], x: node[:x], scale: scale, max_depth: max_depth)
|
|
38
42
|
end
|
|
39
43
|
end
|
|
40
44
|
|
|
41
45
|
def position_cost(nodes, x:, scale:, max_depth:)
|
|
42
|
-
cursor = x
|
|
46
|
+
cursor = x.to_f
|
|
43
47
|
nodes.each do |node|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
+
start = cursor.round
|
|
49
|
+
cursor += node[:subtree_cost] * scale
|
|
50
|
+
node[:cost_x] = start
|
|
51
|
+
node[:cost_w] = cursor.round - start
|
|
52
|
+
position_cost(node[:children], x: node[:cost_x], scale: scale, max_depth: max_depth)
|
|
48
53
|
end
|
|
49
54
|
end
|
|
50
55
|
|
|
@@ -5,7 +5,17 @@ module TokenLens
|
|
|
5
5
|
class Reshaper
|
|
6
6
|
def reshape(nodes)
|
|
7
7
|
nodes = collapse_streaming(nodes)
|
|
8
|
-
|
|
8
|
+
# Process roots iteratively so that human prompts discovered mid-thread
|
|
9
|
+
# (nested inside an assistant chain) are hoisted to the top level rather
|
|
10
|
+
# than stacked as children of the prompt that preceded them.
|
|
11
|
+
@pending_roots = nodes.dup
|
|
12
|
+
result = []
|
|
13
|
+
while @pending_roots.any?
|
|
14
|
+
batch = @pending_roots
|
|
15
|
+
@pending_roots = []
|
|
16
|
+
result += batch.flat_map { |node| process_root(node) }
|
|
17
|
+
end
|
|
18
|
+
result
|
|
9
19
|
end
|
|
10
20
|
|
|
11
21
|
private
|
|
@@ -58,11 +68,17 @@ module TokenLens
|
|
|
58
68
|
# Flatten a linear user→assistant→user(tool_result)→assistant chain into
|
|
59
69
|
# a flat list of assistant siblings, computing marginal_input_tokens deltas.
|
|
60
70
|
# Sidechain children stay nested under the assistant that spawned them.
|
|
61
|
-
|
|
71
|
+
#
|
|
72
|
+
# through_assistant: tracks whether we've passed at least one assistant turn.
|
|
73
|
+
# A human prompt encountered BEFORE any assistant (e.g. a screenshot attached
|
|
74
|
+
# to the same user turn) is treated as transparent — we recurse into its
|
|
75
|
+
# children rather than hoisting it. A human prompt encountered AFTER an
|
|
76
|
+
# assistant is a genuine new conversational turn and gets hoisted.
|
|
77
|
+
def flatten_thread(nodes, prev_input:, through_assistant: false)
|
|
62
78
|
nodes.flat_map do |node|
|
|
63
79
|
t = node[:token]
|
|
64
80
|
if t.role == "user" && !t.is_human_prompt?
|
|
65
|
-
flatten_thread(node[:children], prev_input: prev_input)
|
|
81
|
+
flatten_thread(node[:children], prev_input: prev_input, through_assistant: through_assistant)
|
|
66
82
|
elsif t.role == "assistant"
|
|
67
83
|
marginal = [t.input_tokens - prev_input, 0].max
|
|
68
84
|
compaction = prev_input > 0 && t.input_tokens < prev_input * 0.5
|
|
@@ -77,9 +93,15 @@ module TokenLens
|
|
|
77
93
|
token: t.with(marginal_input_tokens: marginal, is_compaction: compaction),
|
|
78
94
|
children: sidechain
|
|
79
95
|
)
|
|
80
|
-
[updated] + flatten_thread(chain, prev_input: t.input_tokens)
|
|
96
|
+
[updated] + flatten_thread(chain, prev_input: t.input_tokens, through_assistant: true)
|
|
97
|
+
elsif through_assistant
|
|
98
|
+
# Human prompt after an assistant — genuine new turn, hoist to top level.
|
|
99
|
+
@pending_roots << node
|
|
100
|
+
[]
|
|
81
101
|
else
|
|
82
|
-
|
|
102
|
+
# Human prompt before any assistant (consecutive user messages, e.g. an
|
|
103
|
+
# image attachment). Treat as transparent and continue into its children.
|
|
104
|
+
flatten_thread(node[:children], prev_input: prev_input, through_assistant: false)
|
|
83
105
|
end
|
|
84
106
|
end
|
|
85
107
|
end
|
data/lib/token_lens/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: token-lens
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- rob durst
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date: 2026-03-
|
|
10
|
+
date: 2026-03-24 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
13
|
name: thor
|