space-architect 8.0.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +54 -0
- data/README.md +39 -464
- data/lib/space_architect/architect_project.rb +90 -39
- data/lib/space_architect/bug_report.rb +2 -1
- data/lib/space_architect/cli/architect.rb +189 -11
- data/lib/space_architect/cli/research.rb +1 -1
- data/lib/space_architect/cli.rb +9 -14
- data/lib/space_architect/loop_cost.rb +592 -0
- data/lib/space_architect/research/supervisor.rb +9 -2
- data/lib/space_architect/routing.rb +149 -0
- data/lib/space_architect/session_sync/bin_path.rb +1 -1
- data/lib/space_architect/session_sync/cursor.rb +1 -1
- data/lib/space_architect/session_sync/plist.rb +1 -1
- data/lib/{space_core → space_architect}/version.rb +2 -2
- data/lib/space_architect.rb +2 -1
- data/skill/architect/SKILL.md +3 -2
- data/skill/architect-vocabulary/SKILL.md +2 -2
- metadata +20 -177
- data/exe/space +0 -13
- data/exe/src +0 -13
- data/lib/space_architect/cli/space.rb +0 -29
- data/lib/space_architect/cli/src.rb +0 -38
- data/lib/space_core/atomic_write.rb +0 -21
- data/lib/space_core/cli/base_command.rb +0 -19
- data/lib/space_core/cli/build.rb +0 -27
- data/lib/space_core/cli/config.rb +0 -49
- data/lib/space_core/cli/current.rb +0 -16
- data/lib/space_core/cli/help.rb +0 -195
- data/lib/space_core/cli/helpers.rb +0 -115
- data/lib/space_core/cli/init.rb +0 -29
- data/lib/space_core/cli/list.rb +0 -24
- data/lib/space_core/cli/loop_status.rb +0 -47
- data/lib/space_core/cli/new.rb +0 -38
- data/lib/space_core/cli/pack.rb +0 -29
- data/lib/space_core/cli/path.rb +0 -16
- data/lib/space_core/cli/repeatable_options.rb +0 -84
- data/lib/space_core/cli/repo.rb +0 -76
- data/lib/space_core/cli/run.rb +0 -44
- data/lib/space_core/cli/shell.rb +0 -125
- data/lib/space_core/cli/show.rb +0 -21
- data/lib/space_core/cli/status.rb +0 -74
- data/lib/space_core/cli/use.rb +0 -17
- data/lib/space_core/cli.rb +0 -177
- data/lib/space_core/commands.rb +0 -51
- data/lib/space_core/config.rb +0 -152
- data/lib/space_core/errors.rb +0 -14
- data/lib/space_core/git_client.rb +0 -49
- data/lib/space_core/mise_client.rb +0 -37
- data/lib/space_core/oci_builder.rb +0 -56
- data/lib/space_core/oci_packer.rb +0 -103
- data/lib/space_core/oci_runner.rb +0 -89
- data/lib/space_core/paths.rb +0 -58
- data/lib/space_core/repo_reference.rb +0 -19
- data/lib/space_core/repo_resolver.rb +0 -167
- data/lib/space_core/shell_integration.rb +0 -438
- data/lib/space_core/slugger.rb +0 -16
- data/lib/space_core/space.rb +0 -174
- data/lib/space_core/space_store.rb +0 -319
- data/lib/space_core/state.rb +0 -86
- data/lib/space_core/templates/oci/dockerfile.erb +0 -73
- data/lib/space_core/templates/oci/dockerignore.erb +0 -17
- data/lib/space_core/templates/oci/entrypoint.sh.erb +0 -16
- data/lib/space_core/terminal.rb +0 -161
- data/lib/space_core/warnings.rb +0 -13
- data/lib/space_core/xdg.rb +0 -33
- data/lib/space_core.rb +0 -26
- data/lib/space_src/cli/clone.rb +0 -54
- data/lib/space_src/cli/config.rb +0 -65
- data/lib/space_src/cli/daemon.rb +0 -362
- data/lib/space_src/cli/options.rb +0 -21
- data/lib/space_src/cli/org.rb +0 -199
- data/lib/space_src/cli/repo.rb +0 -169
- data/lib/space_src/cli/shell.rb +0 -121
- data/lib/space_src/cli/status.rb +0 -75
- data/lib/space_src/cli/sync.rb +0 -161
- data/lib/space_src/cli.rb +0 -159
- data/lib/space_src/cloner.rb +0 -77
- data/lib/space_src/config/contract.rb +0 -54
- data/lib/space_src/config/duration.rb +0 -79
- data/lib/space_src/config/model.rb +0 -49
- data/lib/space_src/config/store.rb +0 -156
- data/lib/space_src/forge/client.rb +0 -31
- data/lib/space_src/forge/github.rb +0 -98
- data/lib/space_src/launchd/agent.rb +0 -195
- data/lib/space_src/launchd/plist.rb +0 -129
- data/lib/space_src/log_rotator.rb +0 -46
- data/lib/space_src/migration.rb +0 -43
- data/lib/space_src/nav.rb +0 -99
- data/lib/space_src/paths.rb +0 -72
- data/lib/space_src/scm/client.rb +0 -87
- data/lib/space_src/scm/git.rb +0 -232
- data/lib/space_src/scm/status.rb +0 -24
- data/lib/space_src/shell.rb +0 -90
- data/lib/space_src/shell_integration.rb +0 -321
- data/lib/space_src/state/lock.rb +0 -59
- data/lib/space_src/state/store.rb +0 -140
- data/lib/space_src/sync/engine.rb +0 -503
- data/lib/space_src/sync/repo_plan.rb +0 -215
- data/lib/space_src/sync/report.rb +0 -18
- data/lib/space_src/ui/interactive_reporter.rb +0 -280
- data/lib/space_src/ui/json_reporter.rb +0 -39
- data/lib/space_src/ui/mode.rb +0 -68
- data/lib/space_src/ui/plain_reporter.rb +0 -53
- data/lib/space_src/ui/reporter.rb +0 -48
- data/lib/space_src/version.rb +0 -7
- data/lib/space_src.rb +0 -38
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "time"
|
|
5
|
+
require "find"
|
|
6
|
+
require "open3"
|
|
7
|
+
require_relative "harness"
|
|
8
|
+
|
|
9
|
+
module Space::Architect
|
|
10
|
+
# `architect cost` aggregation — the read-only accounting of the loop's spend
|
|
11
|
+
# and speed from the pi session artifacts already on disk. Pure module in the
|
|
12
|
+
# routing.rb house shape: no CLI or terminal dependencies, plain class
|
|
13
|
+
# methods, Space::Core::Error naming paths on failure. The corpus is the slug
|
|
14
|
+
# dir under the pi sessions root (`--` + space path with `/` → `-` + `--`);
|
|
15
|
+
# three artifact shapes are folded into one model: run metas (`*_meta.json`,
|
|
16
|
+
# any depth), session files (any other `*.jsonl` whose first line is a
|
|
17
|
+
# `type: "session"` header — interactive and runner files), and run
|
|
18
|
+
# transcripts (`*_transcript.jsonl`, latency rounds only — their per-message
|
|
19
|
+
# cost belongs to the meta's total, so counting both would double-count).
|
|
20
|
+
# Everything skips malformed files/lines instead of crashing: the sums
|
|
21
|
+
# reflect what parsed. Model identity is normalized to one bare key per
|
|
22
|
+
# model — runs by requestedModel (falling back to model) with a trailing
|
|
23
|
+
# `:level` thinking suffix stripped via Harness.parse_model_suffix, session
|
|
24
|
+
# messages by provider/model — so no row key ever carries a suffix.
|
|
25
|
+
module LoopCost
|
|
26
|
+
# The agent label for session-file rows; a session file carries no agent
|
|
27
|
+
# identity, and only assistant messages carry usage, so the two artifact
|
|
28
|
+
# kinds never double-count each other.
|
|
29
|
+
SESSION_AGENT = "(session)"
|
|
30
|
+
|
|
31
|
+
class << self
|
|
32
|
+
# The pi sessions dir for a space: the space's absolute path with every
|
|
33
|
+
# `/` turned into `-`, wrapped in dashes — the trailing `/` is slugified
|
|
34
|
+
# too, which is why live dirs carry TWO leading and TWO trailing dashes
|
|
35
|
+
# (`/Users/eric` → `--Users-eric--`). Verified against every dir in the
|
|
36
|
+
# live sessions root; the frozen cost-readonly gate's fixture dir name
|
|
37
|
+
# (`--Users-eric-architect-spaces-20261009-pi-port--`) pins the same
|
|
38
|
+
# shape. (The spec's one-line `-` + tr + `-` formula omits the trailing
|
|
39
|
+
# separator and names a dir that does not exist.)
|
|
40
|
+
def slug_dir(space_path)
|
|
41
|
+
"-" + "#{space_path}/".tr("/", "-") + "-"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# `--since` intake: ISO-8601 timestamps, or a bare `YYYY-MM-DD` date —
|
|
45
|
+
# local midnight of that calendar day, since a date with no zone means
|
|
46
|
+
# the local day. Any other shape is user error, raised loud.
|
|
47
|
+
def parse_since(value)
|
|
48
|
+
return nil if value.nil? || value.strip.empty?
|
|
49
|
+
|
|
50
|
+
text = value.strip
|
|
51
|
+
if (m = text.match(/\A(\d{4})-(\d{2})-(\d{2})\z/))
|
|
52
|
+
return Time.new(m[1].to_i, m[2].to_i, m[3].to_i)
|
|
53
|
+
end
|
|
54
|
+
return Time.iso8601(text) if iso8601?(text)
|
|
55
|
+
|
|
56
|
+
raise Space::Core::Error,
|
|
57
|
+
"--since '#{value}' is not an ISO-8601 timestamp or YYYY-MM-DD date"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# The whole report: the frozen `--json` contract shape (plus `iterations`
|
|
61
|
+
# / `grand_total` for --by-iteration and `latency` for --latency). The
|
|
62
|
+
# text views render from this same hash, so text and JSON can never
|
|
63
|
+
# disagree on the numbers.
|
|
64
|
+
def report(space:, sessions_root:, since: nil, by_iteration: false, latency: false)
|
|
65
|
+
root = File.expand_path(sessions_root.to_s)
|
|
66
|
+
slug = slug_dir(space.path)
|
|
67
|
+
slug_path = File.join(root, slug)
|
|
68
|
+
unless File.directory?(slug_path)
|
|
69
|
+
raise Space::Core::Error,
|
|
70
|
+
"no sessions dir for space #{space.id} — expected #{slug_path} (sessions root #{root})"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
floor = parse_since(since)
|
|
74
|
+
corpus = load_corpus(slug_path: slug_path, floor: floor, with_transcripts: latency)
|
|
75
|
+
rows, totals = aggregate(corpus[:runs], corpus[:sessions])
|
|
76
|
+
|
|
77
|
+
report = {
|
|
78
|
+
"space" => space.id,
|
|
79
|
+
"slug_dir" => slug,
|
|
80
|
+
"sessions_root" => root,
|
|
81
|
+
"since" => since,
|
|
82
|
+
"runs" => corpus[:runs].size,
|
|
83
|
+
"sessions" => corpus[:sessions].size,
|
|
84
|
+
"rows" => rows,
|
|
85
|
+
"totals" => totals
|
|
86
|
+
}
|
|
87
|
+
if by_iteration
|
|
88
|
+
buckets = iteration_buckets(corpus[:runs], corpus[:sessions], space)
|
|
89
|
+
report["iterations"] = buckets
|
|
90
|
+
# The buckets partition the filtered corpus exactly, so the grand
|
|
91
|
+
# total IS the corpus total — one accumulation, no re-summing drift.
|
|
92
|
+
report["grand_total"] = report["totals"]
|
|
93
|
+
end
|
|
94
|
+
report["latency"] = latency_rows(corpus[:rounds]) if latency
|
|
95
|
+
report
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Iteration boundaries, in ordinal order: git commit time of freeze_sha
|
|
99
|
+
# in the space repo (authoritative — it exists even when the rehearsal
|
|
100
|
+
# clock drifted), falling back to rehearsal.at, then the earliest
|
|
101
|
+
# dispatched_at. An iteration with none of the three is unusable and
|
|
102
|
+
# yields no window; its span simply merges into the preceding one.
|
|
103
|
+
def iteration_boundaries(space)
|
|
104
|
+
project = space.data["project"].is_a?(Hash) ? space.data["project"] : {}
|
|
105
|
+
entries = project["iterations"].is_a?(Array) ? project["iterations"] : []
|
|
106
|
+
entries.sort_by { |e| e.is_a?(Hash) ? e["ordinal"].to_i : 0 }.filter_map do |entry|
|
|
107
|
+
next unless entry.is_a?(Hash)
|
|
108
|
+
|
|
109
|
+
at = boundary_time(entry, space.path)
|
|
110
|
+
at && { name: entry["name"].to_s, at: at }
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
private
|
|
115
|
+
|
|
116
|
+
def iso8601?(text)
|
|
117
|
+
Time.iso8601(text)
|
|
118
|
+
true
|
|
119
|
+
rescue ArgumentError
|
|
120
|
+
false
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def safe_iso(value)
|
|
124
|
+
Time.iso8601(value) if value.is_a?(String)
|
|
125
|
+
rescue ArgumentError
|
|
126
|
+
nil
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def within_floor?(start, floor)
|
|
130
|
+
floor.nil? || start >= floor
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def numeric(value)
|
|
134
|
+
value.is_a?(Numeric) ? value : 0
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# One malformed line drops only itself — a truncated tail is just the
|
|
138
|
+
# last partial line failing to parse — never the records around it.
|
|
139
|
+
def parse_line(line)
|
|
140
|
+
JSON.parse(line)
|
|
141
|
+
rescue JSON::ParserError
|
|
142
|
+
nil
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Session costs arrive as a Hash `{total: …}`; metas carry a plain
|
|
146
|
+
# number. Both accepted, anything else counts as zero.
|
|
147
|
+
def cost_of(usage)
|
|
148
|
+
cost = usage["cost"]
|
|
149
|
+
return cost["total"].to_f if cost.is_a?(Hash) && cost["total"].is_a?(Numeric)
|
|
150
|
+
return cost.to_f if cost.is_a?(Numeric)
|
|
151
|
+
|
|
152
|
+
0.0
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# One bare key per model: the trailing `:level` thinking suffix appears
|
|
156
|
+
# on metas' requestedModel AND model, and on session models — strip it
|
|
157
|
+
# everywhere so `zai/glm-5.3-flash:high` folds into `zai/glm-5.3-flash`.
|
|
158
|
+
def normalize_model(ref)
|
|
159
|
+
return nil unless ref.is_a?(String) && !ref.strip.empty?
|
|
160
|
+
|
|
161
|
+
Harness.parse_model_suffix(ref.strip)[0]
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def epoch_ms(value)
|
|
165
|
+
Time.at(value / 1000.0) if value.is_a?(Numeric)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def new_usage
|
|
169
|
+
{ input: 0, output: 0, cache_read: 0, cache_write: 0, cost: 0.0 }
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def new_cell
|
|
173
|
+
new_usage.merge(runs: 0, turns: 0)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def add_usage!(cell, usage)
|
|
177
|
+
cell[:input] += usage[:input]
|
|
178
|
+
cell[:output] += usage[:output]
|
|
179
|
+
cell[:cache_read] += usage[:cache_read]
|
|
180
|
+
cell[:cache_write] += usage[:cache_write]
|
|
181
|
+
cell[:cost] += usage[:cost]
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# ---- discovery -------------------------------------------------------
|
|
185
|
+
|
|
186
|
+
def load_corpus(slug_path:, floor:, with_transcripts:)
|
|
187
|
+
runs = []
|
|
188
|
+
sessions = []
|
|
189
|
+
transcript_files = []
|
|
190
|
+
Find.find(slug_path) do |path| # paths:exempt — walks the foreign pi sessions corpus (a slug dir, not the space's own tree); any-depth discovery of *_meta.json and .jsonl session files IS this module's contract
|
|
191
|
+
next unless File.file?(path)
|
|
192
|
+
if path.end_with?("_meta.json")
|
|
193
|
+
run = read_run(path)
|
|
194
|
+
runs << run if run && within_floor?(run[:start], floor)
|
|
195
|
+
elsif path.end_with?(".jsonl")
|
|
196
|
+
header = read_session_header(path)
|
|
197
|
+
if header
|
|
198
|
+
session = read_session(path, header)
|
|
199
|
+
sessions << session if session && within_floor?(session[:start], floor)
|
|
200
|
+
elsif with_transcripts && path.end_with?("_transcript.jsonl")
|
|
201
|
+
transcript_files << path
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
corpus = { runs: runs, sessions: sessions, rounds: [] }
|
|
207
|
+
# Session rounds ride into the latency corpus as per-model tuples (the
|
|
208
|
+
# walker already folds a file's rounds per key); transcripts append
|
|
209
|
+
# below. Both shapes share the corpus — one round definition.
|
|
210
|
+
sessions.each do |session|
|
|
211
|
+
session[:rounds].each do |model, cell|
|
|
212
|
+
corpus[:rounds] <<
|
|
213
|
+
{ model: model, rounds: cell[:count], wall_s: cell[:wall_s], output_tokens: cell[:output_tokens],
|
|
214
|
+
gen_ms: cell[:gen_ms], tool_ms: cell[:tool_ms] }
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
return corpus unless with_transcripts
|
|
218
|
+
|
|
219
|
+
# Join runs by runId from the ALREADY-filtered list: a run dropped by
|
|
220
|
+
# --since must not have its transcript leak back in through the join.
|
|
221
|
+
runs_by_id = runs.filter_map { |r| r[:run_id] && [r[:run_id], r] }.to_h
|
|
222
|
+
transcript_files.each do |path|
|
|
223
|
+
artifact = read_transcript(path, runs_by_id)
|
|
224
|
+
next unless artifact && within_floor?(artifact[:start], floor)
|
|
225
|
+
|
|
226
|
+
corpus[:rounds].concat(artifact[:rounds])
|
|
227
|
+
end
|
|
228
|
+
corpus
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# A meta unusable for row attribution (unparseable, no agent, no model
|
|
232
|
+
# key, no usage, no timestamp) drops out of the sums entirely — there is
|
|
233
|
+
# no row it could land in. The live corpus always carries all of these.
|
|
234
|
+
def read_run(path)
|
|
235
|
+
meta = JSON.parse(File.read(path))
|
|
236
|
+
return nil unless meta.is_a?(Hash)
|
|
237
|
+
|
|
238
|
+
usage = meta["usage"].is_a?(Hash) ? meta["usage"] : nil
|
|
239
|
+
agent = meta["agent"]
|
|
240
|
+
model = normalize_model(meta["requestedModel"] || meta["model"])
|
|
241
|
+
start = epoch_ms(meta["timestamp"])
|
|
242
|
+
return nil unless usage && agent.is_a?(String) && !agent.empty? && model && start
|
|
243
|
+
|
|
244
|
+
{
|
|
245
|
+
agent: agent,
|
|
246
|
+
model: model,
|
|
247
|
+
run_id: meta["runId"],
|
|
248
|
+
start: start,
|
|
249
|
+
input: numeric(usage["input"]),
|
|
250
|
+
output: numeric(usage["output"]),
|
|
251
|
+
cache_read: numeric(usage["cacheRead"]),
|
|
252
|
+
cache_write: numeric(usage["cacheWrite"]),
|
|
253
|
+
cost: cost_of(usage),
|
|
254
|
+
turns: numeric(usage["turns"])
|
|
255
|
+
}
|
|
256
|
+
rescue JSON::ParserError, Errno::ENOENT, Errno::EISDIR
|
|
257
|
+
nil
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# A session file IS any .jsonl whose first line is a type:session header
|
|
261
|
+
# — the check is structural, so transcripts and noise files are excluded
|
|
262
|
+
# by shape, never by name.
|
|
263
|
+
def read_session_header(path)
|
|
264
|
+
File.open(path) do |file|
|
|
265
|
+
line = file.gets
|
|
266
|
+
header = line && JSON.parse(line)
|
|
267
|
+
header.is_a?(Hash) && header["type"] == "session" ? header : nil
|
|
268
|
+
end
|
|
269
|
+
rescue JSON::ParserError, Errno::ENOENT, Errno::EISDIR
|
|
270
|
+
nil
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def read_session(path, header)
|
|
274
|
+
start = safe_iso(header["timestamp"])
|
|
275
|
+
return nil unless start
|
|
276
|
+
|
|
277
|
+
usage = {}
|
|
278
|
+
rounds = {}
|
|
279
|
+
open_round = nil
|
|
280
|
+
File.foreach(path) do |line|
|
|
281
|
+
record = parse_line(line)
|
|
282
|
+
next unless record
|
|
283
|
+
message = record["message"]
|
|
284
|
+
next unless message.is_a?(Hash)
|
|
285
|
+
|
|
286
|
+
role = message["role"]
|
|
287
|
+
ts = safe_iso(record["timestamp"]) || safe_iso(message["timestamp"])
|
|
288
|
+
if role == "user"
|
|
289
|
+
close_round!(rounds, open_round) if open_round
|
|
290
|
+
# A user record without a timestamp cannot anchor a wall time —
|
|
291
|
+
# the round would be unmeasurable, so it opens nothing.
|
|
292
|
+
open_round = ts ? new_round(ts) : nil
|
|
293
|
+
elsif role == "assistant"
|
|
294
|
+
message_usage = message["usage"].is_a?(Hash) ? message["usage"] : nil
|
|
295
|
+
model = session_model_key(message)
|
|
296
|
+
if message_usage && model
|
|
297
|
+
cell = (usage[model] ||= new_usage)
|
|
298
|
+
cell[:input] += numeric(message_usage["input"])
|
|
299
|
+
cell[:output] += numeric(message_usage["output"])
|
|
300
|
+
cell[:cache_read] += numeric(message_usage["cacheRead"])
|
|
301
|
+
cell[:cache_write] += numeric(message_usage["cacheWrite"])
|
|
302
|
+
cell[:cost] += cost_of(message_usage)
|
|
303
|
+
end
|
|
304
|
+
if open_round
|
|
305
|
+
# Generation time is self-contained (durationMs), so it accrues
|
|
306
|
+
# even when a missing ts leaves the round wall unmeasurable.
|
|
307
|
+
open_round[:gen_ms] += numeric(message["durationMs"])
|
|
308
|
+
next unless ts
|
|
309
|
+
open_round[:wall_s] = [ts - open_round[:user_ts], 0.0].max
|
|
310
|
+
open_round[:output_tokens] += numeric(message_usage["output"]) if message_usage
|
|
311
|
+
open_round[:model] = model if model
|
|
312
|
+
end
|
|
313
|
+
elsif role == "toolResult" && open_round
|
|
314
|
+
open_round[:tool_ms] += numeric(message["durationMs"])
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
close_round!(rounds, open_round) if open_round
|
|
318
|
+
{ start: start, usage: usage, rounds: rounds }
|
|
319
|
+
rescue Errno::ENOENT
|
|
320
|
+
# The file vanished mid-read: the round still closes and everything
|
|
321
|
+
# parsed so far stays in the sums.
|
|
322
|
+
close_round!(rounds, open_round) if open_round
|
|
323
|
+
{ start: start, usage: usage, rounds: rounds }
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
def session_model_key(message)
|
|
327
|
+
provider = message["provider"]
|
|
328
|
+
model = message["model"]
|
|
329
|
+
return nil unless provider.is_a?(String) && !provider.empty? && model.is_a?(String) && !model.empty?
|
|
330
|
+
|
|
331
|
+
normalize_model("#{provider}/#{model}")
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# ---- shared round state ---------------------------------------------
|
|
335
|
+
|
|
336
|
+
# One assistant round: user anchor → last assistant before the next
|
|
337
|
+
# user. wall_s is turn wall (includes tool and child-wait time); gen_ms
|
|
338
|
+
# sums the assistants' own durationMs (generation time); tool_ms sums
|
|
339
|
+
# tool execution time (toolResult.durationMs session-side, tool_start →
|
|
340
|
+
# tool_end spans transcript-side).
|
|
341
|
+
def new_round(user_ts)
|
|
342
|
+
{ user_ts: user_ts, wall_s: 0.0, output_tokens: 0, gen_ms: 0, tool_ms: 0, model: nil }
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def close_round!(rounds, round)
|
|
346
|
+
return unless round[:model]
|
|
347
|
+
|
|
348
|
+
cell = (rounds[round[:model]] ||= { count: 0, wall_s: 0.0, output_tokens: 0, gen_ms: 0, tool_ms: 0 })
|
|
349
|
+
absorb_round!(cell, round)
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
def absorb_round!(cell, round)
|
|
353
|
+
cell[:count] += 1
|
|
354
|
+
cell[:wall_s] += round[:wall_s]
|
|
355
|
+
cell[:output_tokens] += round[:output_tokens]
|
|
356
|
+
cell[:gen_ms] += round[:gen_ms]
|
|
357
|
+
cell[:tool_ms] += round[:tool_ms]
|
|
358
|
+
cell
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# Same round state machine as read_session: user opens, the last
|
|
362
|
+
# assistant before the next user (or EOF) closes. A transcript maps to
|
|
363
|
+
# its run meta via runId for the model key — its own record model ids
|
|
364
|
+
# are provider-scoped and NOT stable keys — falling back to the record's
|
|
365
|
+
# own model when no meta resolves (orphan transcripts exist in the wild).
|
|
366
|
+
def read_transcript(path, runs_by_id)
|
|
367
|
+
rounds = []
|
|
368
|
+
open_round = nil
|
|
369
|
+
open_tools = {}
|
|
370
|
+
run_ids = []
|
|
371
|
+
first_ts = nil
|
|
372
|
+
File.foreach(path) do |line|
|
|
373
|
+
record = parse_line(line)
|
|
374
|
+
next unless record.is_a?(Hash)
|
|
375
|
+
|
|
376
|
+
if %w[tool_start tool_end].include?(record["recordType"])
|
|
377
|
+
id = record["toolCallId"]
|
|
378
|
+
ts = record_time(record)
|
|
379
|
+
if record["recordType"] == "tool_start"
|
|
380
|
+
open_tools[id] = ts if id && ts
|
|
381
|
+
elsif id && ts && (start = open_tools.delete(id))
|
|
382
|
+
# Attributed to the round open at tool end — a span crossing a
|
|
383
|
+
# user boundary lands in the round it finished inside. record_time
|
|
384
|
+
# yields Time objects, so the difference is seconds → ms for the
|
|
385
|
+
# round accumulator.
|
|
386
|
+
open_round[:tool_ms] += [(ts - start) * 1000, 0.0].max if open_round
|
|
387
|
+
end
|
|
388
|
+
next
|
|
389
|
+
end
|
|
390
|
+
next unless record["recordType"] == "message"
|
|
391
|
+
|
|
392
|
+
run_ids << record["runId"] if record["runId"].is_a?(String)
|
|
393
|
+
first_ts ||= record_time(record)
|
|
394
|
+
role = record["role"] || (record["message"].is_a?(Hash) && record["message"]["role"])
|
|
395
|
+
ts = record_time(record)
|
|
396
|
+
if role == "user"
|
|
397
|
+
close_transcript_round!(rounds, open_round) if open_round
|
|
398
|
+
# No timestamp → no wall to measure; the round opens nothing.
|
|
399
|
+
open_round = ts ? new_round(ts) : nil
|
|
400
|
+
elsif role == "assistant" && open_round
|
|
401
|
+
open_round[:gen_ms] += numeric(record.dig("message", "durationMs"))
|
|
402
|
+
next unless ts
|
|
403
|
+
usage = record["usage"].is_a?(Hash) ? record["usage"] : nil
|
|
404
|
+
open_round[:wall_s] = [ts - open_round[:user_ts], 0.0].max
|
|
405
|
+
open_round[:output_tokens] += numeric(usage["output"]) if usage
|
|
406
|
+
model = joined_model(record, runs_by_id) || normalize_model(record["model"])
|
|
407
|
+
open_round[:model] = model if model
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
close_transcript_round!(rounds, open_round) if open_round
|
|
411
|
+
return nil unless first_ts
|
|
412
|
+
|
|
413
|
+
meta = run_ids.uniq.size == 1 ? runs_by_id[run_ids.first] : nil
|
|
414
|
+
{ start: meta ? meta[:start] : first_ts, rounds: rounds }
|
|
415
|
+
rescue Errno::ENOENT
|
|
416
|
+
# The file vanished mid-read: the round still closes; rounds parsed
|
|
417
|
+
# so far count.
|
|
418
|
+
close_transcript_round!(rounds, open_round) if open_round
|
|
419
|
+
return nil unless first_ts
|
|
420
|
+
|
|
421
|
+
{ start: first_ts, rounds: rounds }
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
def record_time(record)
|
|
425
|
+
epoch_ms(record["ts"]) || safe_iso(record["timestamp"])
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
def joined_model(record, runs_by_id)
|
|
429
|
+
return nil unless record["runId"].is_a?(String)
|
|
430
|
+
|
|
431
|
+
runs_by_id[record["runId"]]&.fetch(:model, nil)
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
def close_transcript_round!(rounds, round)
|
|
435
|
+
return unless round[:model]
|
|
436
|
+
|
|
437
|
+
rounds << { model: round[:model], rounds: 1 }.merge(
|
|
438
|
+
wall_s: round[:wall_s], output_tokens: round[:output_tokens],
|
|
439
|
+
gen_ms: round[:gen_ms], tool_ms: round[:tool_ms]
|
|
440
|
+
)
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
# ---- aggregation -----------------------------------------------------
|
|
444
|
+
|
|
445
|
+
# Rows keyed agent × model: runs by their meta's agent, session files by
|
|
446
|
+
# the literal (session) label. A session file's per-model row counts the
|
|
447
|
+
# file once and takes its turns from the rounds attributed to that model.
|
|
448
|
+
def aggregate(runs, sessions)
|
|
449
|
+
cells = Hash.new { |hash, key| hash[key] = new_cell }
|
|
450
|
+
runs.each do |run|
|
|
451
|
+
cell = cells[[run[:agent], run[:model]]]
|
|
452
|
+
cell[:runs] += 1
|
|
453
|
+
add_usage!(cell, run)
|
|
454
|
+
cell[:turns] += run[:turns]
|
|
455
|
+
end
|
|
456
|
+
sessions.each do |session|
|
|
457
|
+
session[:usage].each do |model, session_usage|
|
|
458
|
+
cell = cells[[SESSION_AGENT, model]]
|
|
459
|
+
cell[:runs] += 1
|
|
460
|
+
add_usage!(cell, session_usage)
|
|
461
|
+
# Usage can exist without a measurable round (a leading assistant
|
|
462
|
+
# message bills tokens, but no user message opened a round) — its
|
|
463
|
+
# turns are simply the rounds that did parse, possibly 0.
|
|
464
|
+
cell[:turns] += (session[:rounds][model] || {}).fetch(:count, 0)
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
rows = cells.sort_by { |(agent, model), _| [agent, model] }.map do |(agent, model), cell|
|
|
469
|
+
row(agent, model, cell)
|
|
470
|
+
end
|
|
471
|
+
totals_cell = cells.values.reduce(new_cell) do |sum, cell|
|
|
472
|
+
sum[:runs] += cell[:runs]
|
|
473
|
+
sum[:turns] += cell[:turns]
|
|
474
|
+
add_usage!(sum, cell)
|
|
475
|
+
sum
|
|
476
|
+
end
|
|
477
|
+
[rows, totals(totals_cell)]
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
def row(agent, model, cell)
|
|
481
|
+
{ "agent" => agent, "model" => model }.merge(totals(cell))
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
def totals(cell)
|
|
485
|
+
{
|
|
486
|
+
"runs" => cell[:runs],
|
|
487
|
+
"input" => cell[:input],
|
|
488
|
+
"output" => cell[:output],
|
|
489
|
+
"cache_read" => cell[:cache_read],
|
|
490
|
+
"cache_write" => cell[:cache_write],
|
|
491
|
+
"cost" => cell[:cost],
|
|
492
|
+
"turns" => cell[:turns],
|
|
493
|
+
"cache_hit_ratio" => cache_hit_ratio(cell[:cache_read], cell[:input])
|
|
494
|
+
}
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
def cache_hit_ratio(cache_read, input)
|
|
498
|
+
denominator = input + cache_read
|
|
499
|
+
denominator.positive? ? cache_read.fdiv(denominator) : 0.0
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
# ---- iteration buckets ----------------------------------------------
|
|
503
|
+
|
|
504
|
+
def iteration_buckets(runs, sessions, space)
|
|
505
|
+
bounds = iteration_boundaries(space)
|
|
506
|
+
buckets = bounds.map do |bound|
|
|
507
|
+
{ name: bound[:name], starts_at: bound[:at], runs: [], sessions: [] }
|
|
508
|
+
end
|
|
509
|
+
unattributed = { name: "unattributed", starts_at: nil, runs: [], sessions: [] }
|
|
510
|
+
runs.each do |run|
|
|
511
|
+
bucket_for(run[:start], bounds, buckets, unattributed)[:runs] << run
|
|
512
|
+
end
|
|
513
|
+
sessions.each do |session|
|
|
514
|
+
bucket_for(session[:start], bounds, buckets, unattributed)[:sessions] << session
|
|
515
|
+
end
|
|
516
|
+
|
|
517
|
+
(buckets + [unattributed]).map { |bucket| bucket_json(bucket) }
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
# Windows are [boundary_i, boundary_{i+1}) — the last one open-ended;
|
|
521
|
+
# rindex over `>=` lands an artifact exactly on a boundary in the later
|
|
522
|
+
# window. Before the first boundary (or no boundaries at all) is
|
|
523
|
+
# "unattributed".
|
|
524
|
+
def bucket_for(start, bounds, buckets, unattributed)
|
|
525
|
+
index = bounds.rindex { |bound| start >= bound[:at] }
|
|
526
|
+
index ? buckets[index] : unattributed
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
def bucket_json(bucket)
|
|
530
|
+
rows, totals = aggregate(bucket[:runs], bucket[:sessions])
|
|
531
|
+
{
|
|
532
|
+
"name" => bucket[:name],
|
|
533
|
+
"starts_at" => bucket[:starts_at]&.utc&.iso8601,
|
|
534
|
+
"rows" => rows,
|
|
535
|
+
"totals" => totals
|
|
536
|
+
}
|
|
537
|
+
end
|
|
538
|
+
|
|
539
|
+
def boundary_time(entry, space_path)
|
|
540
|
+
sha = entry["freeze_sha"]
|
|
541
|
+
if sha.is_a?(String) && !sha.strip.empty?
|
|
542
|
+
out, _, status = Open3.capture3("git", "-C", space_path.to_s, "show", "-s", "--format=%cI", sha.strip)
|
|
543
|
+
committed = status.success? ? safe_iso(out.strip) : nil
|
|
544
|
+
return committed if committed
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
rehearsal = entry["rehearsal"]
|
|
548
|
+
at = rehearsal.is_a?(Hash) ? rehearsal["at"] : nil
|
|
549
|
+
return safe_iso(at) if safe_iso(at)
|
|
550
|
+
|
|
551
|
+
lanes = entry["lanes"].is_a?(Array) ? entry["lanes"] : []
|
|
552
|
+
lanes.filter_map { |lane| lane.is_a?(Hash) ? safe_iso(lane["dispatched_at"]) : nil }.min
|
|
553
|
+
end
|
|
554
|
+
|
|
555
|
+
# ---- latency ---------------------------------------------------------
|
|
556
|
+
|
|
557
|
+
# Rounds from run transcripts and session files share one shape; wall is
|
|
558
|
+
# clamped at zero because a clock-skewed negative round would otherwise
|
|
559
|
+
# drag the reported means below AC's non-negative floor. Two speeds, two
|
|
560
|
+
# truths (the frozen output_toks_per_s key stays the wall-based figure):
|
|
561
|
+
# turn throughput (output ÷ round wall — tool and child-wait time included,
|
|
562
|
+
# so a bg_wait-heavy coordinator deflates ~3×) and generation speed
|
|
563
|
+
# (output ÷ Σ assistant durationMs — what the model actually emits per
|
|
564
|
+
# second of generating). wait_share is Σ tool time ÷ wall: how much of
|
|
565
|
+
# the turn the model spent waiting on tools.
|
|
566
|
+
def latency_rows(rounds)
|
|
567
|
+
cells = Hash.new { |hash, key| hash[key] = { rounds: 0, wall_s: 0.0, output_tokens: 0, gen_ms: 0, tool_ms: 0 } }
|
|
568
|
+
rounds.each do |round|
|
|
569
|
+
cell = cells[round[:model]]
|
|
570
|
+
cell[:rounds] += round[:rounds]
|
|
571
|
+
cell[:wall_s] += round[:wall_s]
|
|
572
|
+
cell[:output_tokens] += round[:output_tokens]
|
|
573
|
+
cell[:gen_ms] += round[:gen_ms].to_f
|
|
574
|
+
cell[:tool_ms] += round[:tool_ms].to_f
|
|
575
|
+
end
|
|
576
|
+
cells.sort_by { |model, _| model }.map do |model, cell|
|
|
577
|
+
gen_s = cell[:gen_ms] / 1000.0
|
|
578
|
+
{
|
|
579
|
+
"model" => model,
|
|
580
|
+
"rounds" => cell[:rounds],
|
|
581
|
+
"wall_s" => cell[:wall_s],
|
|
582
|
+
"mean_wall_s" => cell[:rounds].positive? ? cell[:wall_s] / cell[:rounds] : 0.0,
|
|
583
|
+
"output_tokens" => cell[:output_tokens],
|
|
584
|
+
"output_toks_per_s" => cell[:wall_s].positive? ? cell[:output_tokens] / cell[:wall_s] : 0.0,
|
|
585
|
+
"gen_toks_per_s" => gen_s.positive? ? cell[:output_tokens] / gen_s : 0.0,
|
|
586
|
+
"wait_share" => cell[:wall_s].positive? ? cell[:tool_ms] / 1000.0 / cell[:wall_s] : 0.0
|
|
587
|
+
}
|
|
588
|
+
end
|
|
589
|
+
end
|
|
590
|
+
end
|
|
591
|
+
end
|
|
592
|
+
end
|
|
@@ -21,8 +21,15 @@ module Space::Architect
|
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
# Dispatch each prompt file as a detached pi child. Returns array of Run
|
|
24
|
-
# objects (non-blocking).
|
|
25
|
-
|
|
24
|
+
# objects (non-blocking). Researchers are NOT loop roles — architect.yaml
|
|
25
|
+
# never routes them — so the pi default is the researcher last resort, made
|
|
26
|
+
# loud here (silent whenever an explicit --model supplies the model).
|
|
27
|
+
def dispatch(prompts, model: nil, max_turns: DEFAULT_MAX_TURNS, err: $stderr)
|
|
28
|
+
if model.nil?
|
|
29
|
+
err.puts "research: researchers are not architect.yaml-routed (loop roles only) — " \
|
|
30
|
+
"no --model given, using default '#{DEFAULT_MODEL}'"
|
|
31
|
+
model = DEFAULT_MODEL
|
|
32
|
+
end
|
|
26
33
|
prompts.map { |path| dispatch_one(Pathname.new(path), model: model, max_turns: max_turns) }
|
|
27
34
|
end
|
|
28
35
|
|