agent-council 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1220 @@
1
+ # frozen_string_literal: true
2
+
3
+ # AgentCouncil — harness-agnostic orchestration: naming, run-directory
4
+ # layout, per-agent subprocess execution, and the CLI entry point.
5
+ # Harness-specific behavior lives under AgentCouncil::Harnesses (e.g.
6
+ # AgentCouncil::Harnesses::PiHarness); HARNESS below is the single swap
7
+ # point.
8
+
9
+ require 'json'
10
+ require 'yaml'
11
+ # Psych requires stringio lazily, on first emit. On JRuby, worker threads
12
+ # emitting front matter at once can then see a half-loaded StringIO
13
+ # (ArgumentError from StringIO.new), so load it before any threads start.
14
+ require 'stringio'
15
+ require 'open3'
16
+ require 'fileutils'
17
+ require 'tmpdir'
18
+ require 'optparse'
19
+ require 'shellwords'
20
+
21
+ require_relative 'agent_council/version'
22
+ require_relative 'agent_council/platform'
23
+ require_relative 'agent_council/harnesses/pi_harness'
24
+ require_relative 'agent_council/worker_pool'
25
+
26
+ module AgentCouncil
27
+ # Runtime data files (prompt templates) ship inside lib/, resolved
28
+ # relative to this file, so the gem is self-contained: what gets loaded at runtime never depends on which
29
+ # files happen to survive the gemspec's file list. PROJECT_ROOT remains
30
+ # only for documentation references (e.g. the local README path in help
31
+ # text), not for runtime data.
32
+ PROJECT_ROOT = File.expand_path('..', __dir__)
33
+ EXAMPLES_DIR = File.join(PROJECT_ROOT, 'examples')
34
+ TEMPLATES_DIR = File.join(__dir__, 'agent_council', 'templates')
35
+ CONSOLIDATE_PROMPT_PATH = File.join(TEMPLATES_DIR, 'consolidate_prompt.md')
36
+ ANALYST_FORMAT_PATH = File.join(TEMPLATES_DIR, 'analyst_format.md')
37
+
38
+ # The harness every model runs through; the single swap point. A second
39
+ # harness would be another module under AgentCouncil::Harnesses,
40
+ # assigned here. Its usage(stdout) returns token counts in the shape
41
+ # documented on PiHarness.usage, or nil when the harness reports none.
42
+ HARNESS = AgentCouncil::Harnesses::PiHarness
43
+
44
+ # After Ctrl-C, how long agents get to exit after SIGTERM before they're
45
+ # killed (see WorkerPool).
46
+ INTERRUPT_GRACE_SECONDS = 5
47
+
48
+ # All text read or captured (prompts, templates, reports, harness output)
49
+ # is UTF-8, whatever the locale's default external encoding (US-ASCII
50
+ # under LANG=C or with LANG unset).
51
+ ENCODING = Encoding::UTF_8
52
+
53
+ # By default, reports go under the current working directory — i.e.
54
+ # alongside the repository being analyzed — so the tool can be run from
55
+ # any project. The name says which tool wrote them, since they land in
56
+ # someone else's tree.
57
+ DEFAULT_REPORTS_DIR_NAME = 'agent-council-reports'
58
+
59
+ # The reports root: the --output-dir value if given (relative paths
60
+ # resolve against the cwd), else ./agent-council-reports.
61
+ module_function def reports_dir(explicit = nil)
62
+ File.expand_path(explicit || DEFAULT_REPORTS_DIR_NAME)
63
+ end
64
+
65
+ # Config file names picked up when no CONFIG_FILE argument is given: a
66
+ # visible per-project file in the current working directory, and a hidden
67
+ # (dotfile) per-user file in the home directory.
68
+ CWD_CONFIG_NAME = 'agent-council.yml'
69
+ HOME_CONFIG_NAME = '.agent-council.yml'
70
+
71
+ # Shell-style arguments from the environment, inserted before ARGV.
72
+ ENV_OPTS_VAR = 'AGENT_COUNCIL_OPTS'
73
+
74
+ # The explicit CONFIG_FILE argument if given, else ./agent-council.yml if
75
+ # it exists, else ~/.agent-council.yml if it exists; nil when none apply.
76
+ # Only the cwd is checked (no upward walk).
77
+ module_function def resolve_config_path(explicit)
78
+ return explicit if explicit
79
+
80
+ candidates = [File.join(Dir.pwd, CWD_CONFIG_NAME)]
81
+ begin
82
+ home = Dir.home
83
+ candidates << File.join(home, HOME_CONFIG_NAME) unless home.empty?
84
+ rescue ArgumentError
85
+ # No resolvable home directory; only the cwd candidate applies.
86
+ end
87
+ candidates.find { |path| File.file?(path) }
88
+ end
89
+
90
+ # An absolute path with the home directory abbreviated to ~, for recording
91
+ # where an input came from without putting the username in shared reports.
92
+ module_function def display_source_path(path)
93
+ path = File.expand_path(path)
94
+ home = begin
95
+ File.expand_path(Dir.home)
96
+ rescue ArgumentError
97
+ nil
98
+ end
99
+ return path if home.nil? || home.empty? || home == File::SEPARATOR
100
+ return '~' if path == home
101
+
102
+ path.start_with?(home + File::SEPARATOR) ? "~#{path.delete_prefix(home)}" : path
103
+ end
104
+
105
+ # Make an identifier safe for a filename component.
106
+ module_function def sanitize(part)
107
+ part.to_s.gsub(/[^A-Za-z0-9._-]+/, '-').gsub(/\A[-.]+|[-.]+\z/, '')
108
+ end
109
+
110
+ # Meaningful query slug, derived from the prompt file name. The extension
111
+ # and then a trailing -prompt or _prompt (in any case) are stripped.
112
+ module_function def query_slug(prompt_path)
113
+ name = File.basename(prompt_path, File.extname(prompt_path)).sub(/[-_]prompt\z/i, '')
114
+ slug = sanitize(name).downcase
115
+ slug.empty? ? 'prompt' : slug
116
+ end
117
+
118
+ # <run_stamp>_<query-slug>[_<index>]_<harness>_<provider>_<model>.md — the
119
+ # full timestamp and slug live in the filename so reports stay
120
+ # self-describing when copied out of the reports tree. index (an agent's
121
+ # 1-based position in the config's agent list) disambiguates repeated
122
+ # provider/model entries — e.g. the same model configured twice for two
123
+ # independent analyses — without depending on it being unique. It sits before the harness so that once more than one harness
124
+ # exists, filenames still sort in config/agent order first, rather than
125
+ # grouping by harness.
126
+ module_function def report_filename(file_prefix, provider, model, index: nil)
127
+ parts = [file_prefix]
128
+ parts << format('%02d', index) if index
129
+ parts << HARNESS::NAME
130
+ parts += [sanitize(provider), sanitize(model)]
131
+ "#{parts.join('_')}.md"
132
+ end
133
+
134
+ # Returns [label, ok, detail, usage]; never raises. usage is the run's
135
+ # token usage (see HARNESS), or nil; failed runs report it too, since
136
+ # their tokens were still spent.
137
+ # index: the agent's 1-based position in the config's agent list, used to
138
+ # disambiguate repeated provider/model entries in the filename and label;
139
+ # filename: overrides the output name; meta: becomes YAML front matter,
140
+ # which identifies the report; what was measured about its run (duration
141
+ # and usage) goes in run.json instead (see write_run_file);
142
+ # header: markdown prepended to the report body (or a callable receiving
143
+ # the run's fields, 'duration' (HH:MM:SS) and 'usage', and returning
144
+ # it). If the response begins
145
+ # with an H1 of its own, it is dropped: the header leads the body (and
146
+ # carries the title, in consolidated reports).
147
+ # pool: the WorkerPool to spawn the agent through, so Ctrl-C can stop it;
148
+ # without one it's spawned untracked.
149
+ module_function def run_agent(agent, prompt, run_dir, file_prefix, index: nil, filename: nil, meta: {},
150
+ header: nil, pool: nil)
151
+ provider = agent['provider'] if agent.is_a?(Hash)
152
+ model = agent['model'] if agent.is_a?(Hash)
153
+ label = agent_label(provider, model, index)
154
+ return [label, false, "config entry needs 'provider' and 'model'"] if provider.nil? || model.nil?
155
+
156
+ # Removed in the ensure below rather than via Dir.mktmpdir's block form,
157
+ # with errors ignored: a child that is still exiting (e.g. one just
158
+ # killed after the Ctrl-C grace period) can still be writing here
159
+ # (node's compile cache, say). That must not surface as an error.
160
+ scratch_dir = Dir.mktmpdir('agent-council-')
161
+ argv = HARNESS.argv(provider, model, scratch_dir)
162
+ (stdout, stderr, status), elapsed = timed do
163
+ (pool || Open3).capture3({ 'TMPDIR' => scratch_dir }, *argv, stdin_data: prompt)
164
+ end
165
+ # Harness output is UTF-8. WorkerPool#capture3 already returns it as such,
166
+ # but Open3.capture3 (used without a pool) returns the locale's encoding.
167
+ stdout, stderr = [stdout, stderr].map { |output| String.new(output, encoding: ENCODING) }
168
+ duration = format_duration(elapsed)
169
+ usage = HARNESS.usage(stdout)
170
+
171
+ unless status.success?
172
+ detail = stderr.lines.map(&:strip).reject(&:empty?).first || 'no error output'
173
+ return [label, false, "#{HARNESS::NAME} exited #{status.exitstatus}: #{detail}", usage]
174
+ end
175
+
176
+ error = HARNESS.error_message(stdout)
177
+ if error
178
+ summary = error.gsub(/\s+/, ' ').slice(0, 300)
179
+ return [label, false, "#{HARNESS::NAME} exited 0 but the run failed: #{summary}", usage]
180
+ end
181
+
182
+ text = HARNESS.final_response(stdout)
183
+ if text.nil? || text.strip.empty?
184
+ detail =
185
+ if HARNESS.assistant_messages(stdout).any?
186
+ 'final assistant message had no text content (model may have answered in reasoning only)'
187
+ else
188
+ "no assistant message in #{HARNESS::NAME} event stream"
189
+ end
190
+ return [label, false, "#{HARNESS::NAME} exited 0 but #{detail}", usage]
191
+ end
192
+
193
+ filename ||= report_filename(file_prefix, provider, model, index: index)
194
+ run_fields = { 'duration' => duration, 'usage' => usage }.compact
195
+ if header
196
+ header = header.call(run_fields) if header.respond_to?(:call)
197
+ text = text.sub(/\A\s*#(?!#).+\n+/, '')
198
+ text = "#{header}\n#{text}"
199
+ end
200
+ content = with_front_matter(meta.merge('harness' => HARNESS::NAME,
201
+ 'provider' => provider,
202
+ 'model' => model), text)
203
+ # Exclusive create ('x'): never replaces a report, e.g. one written by
204
+ # a concurrent --consolidate with the same model; that is a failure.
205
+ File.write(File.join(run_dir, filename), content, mode: 'wx')
206
+ [label, true, filename, usage]
207
+ rescue => e
208
+ [label, false, "#{e.class}: #{e.message}", usage]
209
+ ensure
210
+ FileUtils.rm_rf(scratch_dir) if scratch_dir
211
+ end
212
+
213
+ # Creates a new run dir in day_dir and returns its name: base, or base-2,
214
+ # base-3, ... if that is taken. Dir.mkdir fails if the dir exists, so the
215
+ # claim is atomic: runs started in the same second (or whose local time
216
+ # repeats, e.g. when clocks fall back) never share a dir.
217
+ module_function def claim_run_dir(day_dir, base)
218
+ FileUtils.mkdir_p(day_dir)
219
+ (1..).each do |n|
220
+ name = n == 1 ? base : "#{base}-#{n}"
221
+ Dir.mkdir(File.join(day_dir, name))
222
+ return name
223
+ rescue Errno::EEXIST
224
+ next
225
+ end
226
+ end
227
+
228
+ # The run dir's subdir holding the analysts' reports.
229
+ REVIEWS_DIR_NAME = 'reviews'
230
+
231
+ # A run's analyst reports as [source_label, body, fields] triples, sorted
232
+ # by filename; fields is the report's front matter. The dir is the run's
233
+ # reviews/ subdir; provider/model labels come from the YAML front matter
234
+ # run_agent writes — not from filename parsing. Files without parseable
235
+ # front matter naming a provider and model are not ours and are skipped.
236
+ module_function def collect_reports(reviews_dir)
237
+ # base: keeps reviews_dir from being read as a pattern, so a path with
238
+ # glob characters (e.g. "[") still finds its reports.
239
+ Dir.glob('*.md', base: reviews_dir).filter_map do |name|
240
+ fields, body = parse_front_matter(File.read(File.join(reviews_dir, name), encoding: ENCODING))
241
+ next unless fields && fields['provider'] && fields['model']
242
+
243
+ [agent_label(fields['provider'], fields['model'], fields['agent_index']), body, fields]
244
+ end
245
+ end
246
+
247
+ # --- report front matter ---
248
+
249
+ module_function def with_front_matter(fields, body)
250
+ header = YAML.dump(fields.transform_keys(&:to_s)).delete_prefix("---\n").chomp
251
+ # Documented format: ISO 8601 timestamps, with or without a UTC offset,
252
+ # are unquoted (parse as Time).
253
+ iso = /\d{4}-\d\d-\d\dT[\d:]+(?:Z|[+-]\d\d:\d\d)?/
254
+ header = header.gsub(/^(timestamp|consolidated_at): ['"](#{iso})['"]$/, '\\1: \\2')
255
+ <<~TEXT
256
+ ---
257
+ #{header}
258
+ ---
259
+
260
+ #{body}
261
+ TEXT
262
+ end
263
+
264
+ # [fields, body] from a report file, or [nil, nil] if it has no YAML
265
+ # front matter block (or the block cannot be safe-loaded).
266
+ module_function def parse_front_matter(text)
267
+ match = text.match(/\A---\n(.+?)\n---\n\n?/m)
268
+ return [nil, nil] unless match
269
+
270
+ fields =
271
+ begin
272
+ YAML.safe_load(match[1], permitted_classes: [Time])
273
+ rescue Psych::Exception
274
+ nil
275
+ end
276
+ return [nil, nil] unless fields.is_a?(Hash)
277
+
278
+ [fields, match.post_match]
279
+ end
280
+
281
+ # The exact bytes piped to each analyst harness: the user's prompt, less
282
+ # one trailing newline, then a blank line and the report-format addendum.
283
+ module_function def compose_analyst_prompt(prompt, addendum)
284
+ "#{prompt.chomp}\n\n#{addendum}"
285
+ end
286
+
287
+ # An agent's label in progress lines, report sources, and consolidation
288
+ # prompts: "#<index> <provider> / <model>", or without the index when
289
+ # there is none (the consolidator). missing_agents matches reports by it.
290
+ module_function def agent_label(provider, model, index = nil)
291
+ plain = "#{provider} / #{model}"
292
+ index ? "##{index} #{plain}" : plain
293
+ end
294
+
295
+ # The consolidator's substantive prompt: the instructions template, the
296
+ # original prompt every analyst answered, all reports with their source
297
+ # labels, and the agents without a report, as [label, reason] pairs; the
298
+ # consolidator can't otherwise know about those.
299
+ module_function def consolidation_prompt(original_prompt, reports, failures = [])
300
+ template = File.read(CONSOLIDATE_PROMPT_PATH, encoding: ENCODING)
301
+ body = reports.map do |label, text, fields|
302
+ harness = (fields && fields['harness']) || HARNESS::NAME
303
+ "## Report from #{label}\n\nHarness: #{harness}\n\n#{text}"
304
+ end.join("\n\n---\n\n")
305
+ failed = failures.empty? ? 'None.' : failures.map { |label, reason| "- #{label}: #{reason}" }.join("\n")
306
+ <<~MARKDOWN
307
+ #{template}
308
+
309
+ ---
310
+
311
+ # Original prompt (given identically to every analyst)
312
+
313
+ #{original_prompt}
314
+
315
+ ---
316
+
317
+ # Analyst reports
318
+
319
+ #{body}
320
+
321
+ ---
322
+
323
+ # Agents without a report
324
+
325
+ #{failed}
326
+ MARKDOWN
327
+ end
328
+
329
+ # The consolidator's output filename; lives at the top level of the run
330
+ # dir, and stays self-describing when copied out of the tree.
331
+ # part: 'findings' or 'meta' for the files a consolidated report is split
332
+ # into (see split_consolidated_report); none for the report as written.
333
+ module_function def consolidated_filename(file_prefix, provider, model, part = nil)
334
+ suffix = part ? "_#{part}" : ''
335
+ "#{file_prefix}_consolidated_#{HARNESS::NAME}_#{sanitize(provider)}_#{sanitize(model)}#{suffix}.md"
336
+ end
337
+
338
+ # The line the consolidator is asked to write between its findings and
339
+ # the sections describing the consultation (see consolidate_prompt.md).
340
+ CONSOLIDATED_META_MARKER = '=== META ==='
341
+
342
+ # An analyst's label with its harness: "#<index> <harness> / <provider> /
343
+ # <model>", for the findings file's legend.
344
+ module_function def full_label(fields)
345
+ index = fields['agent_index']
346
+ parts = [fields['harness'] || HARNESS::NAME, fields['provider'], fields['model']].join(' / ')
347
+ index ? "##{index} #{parts}" : parts
348
+ end
349
+
350
+ # [[findings, meta], nil] if text can be split on its marker line, else
351
+ # [nil, why]: a split that can't be trusted isn't made. The marker must occur once, with text on both sides and the
352
+ # Agent Consultants heading opening the second (the template puts that
353
+ # section first), and not inside a code fence, so that a marker the
354
+ # findings quote is not taken for the real one.
355
+ module_function def consolidated_split(text)
356
+ parts = text.split(/^#{Regexp.escape(CONSOLIDATED_META_MARKER)}[ \t\r]*$\n?/, -1)
357
+ line = "#{CONSOLIDATED_META_MARKER} line"
358
+ return [nil, "it has no #{line}"] if parts.size == 1
359
+ return [nil, "it has more than one #{line}"] if parts.size > 2
360
+ return [nil, "there is no text on one side of the #{line}"] if parts.any? { |part| part.strip.empty? }
361
+ unless parts.last.match?(/\A\s*## Agent Consultants/i)
362
+ return [nil, "the #{line} is not followed by an Agent Consultants section"]
363
+ end
364
+ return [nil, "the #{line} is inside a code fence"] if inside_code_fence?(parts.first)
365
+
366
+ [parts, nil]
367
+ end
368
+
369
+ # True if text ends inside a fenced code block, going by CommonMark's
370
+ # rules for top-level fences: three or more backticks or tildes, closed by
371
+ # a fence of the same character at least as long. A backtick fence's info
372
+ # string can't contain a backtick. Fences nested in block quotes or deeply
373
+ # indented list items aren't tracked.
374
+ module_function def inside_code_fence?(text)
375
+ open_fence = nil
376
+ text.each_line do |line|
377
+ fence = line[/\A {0,3}(`{3,}|~{3,})/, 1] or next
378
+ if open_fence.nil?
379
+ open_fence = fence unless fence.start_with?('`') && line.delete_prefix(fence).include?('`')
380
+ elsif fence[0] == open_fence[0] && fence.size >= open_fence.size && line.strip == fence
381
+ open_fence = nil
382
+ end
383
+ end
384
+ !open_fence.nil?
385
+ end
386
+
387
+ # Splits the consolidated report written to filename (the meta file's
388
+ # name: run_agent creates it exclusively, so the name stays taken from
389
+ # then on and concurrent consolidations of the same configuration can't
390
+ # both succeed) in run_dir into a findings file, free of front matter and
391
+ # tables so it can be pasted into a chat as is, and the meta file with the
392
+ # rest, rewritten in place. head: the header run_agent put before the
393
+ # model's text, which stays in the meta file; labels: the analysts' full
394
+ # labels, for the findings file's legend; info: lines (without bullets)
395
+ # identifying the run, for its top. Returns the findings filename and the
396
+ # meta filename, or, if the report can't be split or the split files
397
+ # can't be written, the report under the unsplit name (see
398
+ # consolidated_filename) and nil, with a warning.
399
+ module_function def split_consolidated_report(run_dir, filename, file_prefix, provider, model, head,
400
+ labels, info = [])
401
+ path = File.join(run_dir, filename)
402
+ unsplit_name = consolidated_filename(file_prefix, provider, model)
403
+ findings_name = consolidated_filename(file_prefix, provider, model, 'findings')
404
+ written = []
405
+ content = File.read(path, encoding: ENCODING)
406
+ front = content[/\A---\n.+?\n---\n\n?/m] || ''
407
+ body = content.delete_prefix(front)
408
+ # The header run_agent put before the model's text must still be there,
409
+ # or it would end up in the findings file.
410
+ if head && !body.start_with?("#{head}\n")
411
+ warn "Could not split #{relative_path(path)}: its header is not what was written"
412
+ return keep_unsplit(run_dir, filename, unsplit_name)
413
+ end
414
+ parts, problem = consolidated_split(head ? body.delete_prefix("#{head}\n") : body)
415
+ unless parts
416
+ warn "Not splitting #{relative_path(path)}: #{problem}"
417
+ return keep_unsplit(run_dir, filename, unsplit_name)
418
+ end
419
+
420
+ findings, meta = parts
421
+ about = info.map { |line| "- #{line}\n" }.join
422
+ legend = labels.map { |label| "- #{label}" }.join("\n")
423
+ # Recorded once the file exists, even if writing it then fails, so
424
+ # cleanup removes a partial file; one that was already there (the
425
+ # exclusive create raises) is not ours to remove.
426
+ File.open(File.join(run_dir, findings_name), 'wx') do |file|
427
+ written << findings_name
428
+ file.write("# Consolidated Findings\n\n#{about}\nAgents consulted (agent number, harness, " \
429
+ "provider, model):\n\n#{legend}\n\n#{findings}")
430
+ end
431
+ # Replaced atomically, so the whole report survives a failed write.
432
+ temp = "#{path}.tmp"
433
+ written << File.basename(temp)
434
+ File.write(temp, "#{front}#{head}\n#{meta.lstrip}")
435
+ File.rename(temp, path)
436
+ [findings_name, filename]
437
+ rescue => e
438
+ # A failed split must not fail the consolidation, whose model run is
439
+ # already paid for and whose report is intact.
440
+ warn "Could not split #{relative_path(path)}: #{e.message}"
441
+ written.each { |name| FileUtils.rm_f(File.join(run_dir, name)) }
442
+ keep_unsplit(run_dir, filename, unsplit_name)
443
+ end
444
+
445
+ # Renames the report filename to unsplit_name, unless that is taken (then
446
+ # it stays as it is). Returns the name it ends up under and nil, as
447
+ # split_consolidated_report does for an unsplit report.
448
+ module_function def keep_unsplit(run_dir, filename, unsplit_name)
449
+ target = File.join(run_dir, unsplit_name)
450
+ if File.exist?(target)
451
+ warn "#{relative_path(target)} exists; leaving the report as #{filename}"
452
+ return [filename, nil]
453
+ end
454
+
455
+ File.rename(File.join(run_dir, filename), target)
456
+ [unsplit_name, nil]
457
+ rescue SystemCallError => e
458
+ warn "Could not rename #{filename} to #{unsplit_name} (#{e.message}); leaving the report as #{filename}"
459
+ [filename, nil]
460
+ end
461
+
462
+ # Markdown table of the report's main front-matter fields and its run's
463
+ # duration and usage, leading the body (under the title, in consolidated
464
+ # reports, followed by sources_table).
465
+ module_function def byline_table(meta, provider, model)
466
+ fields = {
467
+ 'role' => meta['role'],
468
+ 'agent' => (meta['agent_index'] ? "##{meta['agent_index']}" : nil),
469
+ 'query' => meta['query'],
470
+ 'harness' => HARNESS::NAME,
471
+ 'provider' => provider,
472
+ 'model' => model,
473
+ 'timestamp' => meta['timestamp'],
474
+ 'consolidated_at' => meta['consolidated_at'],
475
+ 'duration' => meta['duration'],
476
+ 'usage' => format_usage(meta['usage']),
477
+ }.compact
478
+ rows = fields.map { |field, value| "| #{field} | #{value} |" }
479
+ <<~TABLE
480
+ | Field | Value |
481
+ | :-------- | :---------------------------- |
482
+ #{rows.join("\n")}
483
+ TABLE
484
+ end
485
+
486
+ # Markdown table of a consolidated report's sources, one row per label in
487
+ # sources, with its run's duration and usage from source_runs (label =>
488
+ # run.json entry); - where unknown. Input counts cached input too.
489
+ module_function def sources_table(sources, source_runs)
490
+ rows = sources.map do |label|
491
+ run = source_runs[label] || {}
492
+ seconds = run['duration_seconds']
493
+ duration = seconds.is_a?(Numeric) ? format_duration(seconds) : '-'
494
+ "| #{[label, duration, *usage_cells(run['usage'])].join(' | ')} |"
495
+ end
496
+ <<~TABLE
497
+ ### Sources
498
+
499
+ | Source | Duration | Input | Cached | Output | Cost |
500
+ | :----- | -------: | ----: | -----: | -----: | ---: |
501
+ #{rows.join("\n")}
502
+ TABLE
503
+ end
504
+
505
+ # Runs the configured consolidator model over a run's reports and writes
506
+ # its report into the same run dir. Returns as run_agent, except that
507
+ # detail is the findings file's name when the report was split (see
508
+ # split_consolidated_report) and the meta file's name follows usage;
509
+ # never raises.
510
+ # analyst_runs: the run's run.json analyst entries, for the sources
511
+ # table's durations and usage. failures: as for consolidation_prompt.
512
+ # pool: as for run_agent.
513
+ module_function def run_consolidation(consolidator, original_prompt, run_dir, file_prefix,
514
+ meta: {}, prompt_path: nil, failures: [], analyst_runs: [], pool: nil)
515
+ provider = consolidator['provider'] if consolidator.is_a?(Hash)
516
+ model = consolidator['model'] if consolidator.is_a?(Hash)
517
+ return ['consolidator', false, "config entry needs 'provider' and 'model'"] if provider.nil? || model.nil?
518
+
519
+ reports = collect_reports(File.join(run_dir, REVIEWS_DIR_NAME))
520
+ if reports.empty?
521
+ return [agent_label(provider, model), false,
522
+ "no reports found in #{relative_path(File.join(run_dir, REVIEWS_DIR_NAME))}"]
523
+ end
524
+
525
+ prompt = consolidation_prompt(original_prompt, reports, failures)
526
+ source_runs = analyst_runs.grep(Hash)
527
+ .to_h { |run| [agent_label(run['provider'], run['model'], run['agent_index']), run] }
528
+ # timestamp is the run's start; consolidated_at is this consolidation's,
529
+ # which is later when done by --consolidate.
530
+ meta = meta.merge('role' => 'consolidator', 'consolidated_at' => iso_timestamp(Time.now),
531
+ 'sources' => reports.map(&:first))
532
+ prompt_line = prompt_path ? "### Prompt: #{prompt_path}\n\n" : ''
533
+ head = nil
534
+ header = ->(run_fields) do
535
+ head = "# Consolidated Report\n\n#{prompt_line}#{byline_table(meta.merge(run_fields), provider,
536
+ model)}\n" \
537
+ "#{sources_table(meta['sources'], source_runs)}"
538
+ end
539
+ label, ok, detail, usage = run_agent(consolidator, prompt, run_dir, file_prefix,
540
+ filename: consolidated_filename(file_prefix, provider, model, 'meta'),
541
+ header: header,
542
+ meta: meta,
543
+ pool: pool)
544
+ if ok
545
+ labels = reports.map { |_, _, fields| full_label(fields) }
546
+ info = ["Run: #{file_prefix}", prompt_path && "Prompt: #{File.basename(prompt_path)}",
547
+ "Consolidated by: #{HARNESS::NAME} / #{provider} / #{model}, #{meta['consolidated_at']}"].compact
548
+ detail, meta_name = split_consolidated_report(run_dir, detail, file_prefix, provider, model, head,
549
+ labels, info)
550
+ end
551
+ [label, ok, detail, usage, meta_name]
552
+ rescue => e
553
+ [agent_label(provider, model), false, "#{e.class}: #{e.message}"]
554
+ end
555
+
556
+ module_function def relative_path(path)
557
+ path = File.expand_path(path)
558
+ cwd = File.expand_path(Dir.pwd)
559
+ return path unless path == cwd || path.start_with?(cwd + File::SEPARATOR)
560
+
561
+ path.delete_prefix(cwd + File::SEPARATOR)
562
+ end
563
+
564
+ # The front matter's timestamp format: ISO 8601 with the UTC offset.
565
+ module_function def iso_timestamp(time)
566
+ time.strftime('%Y-%m-%dT%H:%M:%S%:z')
567
+ end
568
+
569
+ # Whole seconds as HH:MM:SS (hours may exceed 99); rounds to the nearest second.
570
+ module_function def format_duration(seconds)
571
+ total = seconds.round
572
+ format('%02d:%02d:%02d', total / 3600, total % 3600 / 60, total % 60)
573
+ end
574
+
575
+ # Token usage (see HARNESS) as e.g. "1.2M in (900.0k cached) / 30.5k out,
576
+ # $0.85", where "in" counts all input, cached or not. The cost is left out
577
+ # when unknown. nil for nil usage.
578
+ module_function def format_usage(usage)
579
+ input, cached, output, cost = usage_figures(usage)
580
+ return nil unless input
581
+
582
+ cached_note = " (#{format_token_count(cached)} cached)" if cached.positive?
583
+ text = "#{format_token_count(input)} in#{cached_note} / #{format_token_count(output)} out"
584
+ cost ? "#{text}, #{format_cost(cost)}" : text
585
+ end
586
+
587
+ # Display cells for a usage's input (cached included), cached input,
588
+ # output, and cost; - for each unknown one.
589
+ module_function def usage_cells(usage)
590
+ input, cached, output, cost = usage_figures(usage)
591
+ counts = [input, cached, output].map { |count| count ? format_token_count(count) : '-' }
592
+ [*counts, cost ? format_cost(cost) : '-']
593
+ end
594
+
595
+ # [input (cached or not), cached input, output, cost (nil when unknown)]
596
+ # from a usage hash (see HARNESS), or nil for unknown usage.
597
+ module_function def usage_figures(usage)
598
+ return nil unless usage.is_a?(Hash)
599
+
600
+ cached = usage['cache_read'].to_i
601
+ cost = usage['cost'] if usage['cost'].is_a?(Numeric) && usage['cost'].positive?
602
+ [usage['input'].to_i + cached + usage['cache_write'].to_i, cached, usage['output'].to_i, cost]
603
+ end
604
+
605
+ # 950, 45.2k, 1.2M.
606
+ module_function def format_token_count(count)
607
+ return count.to_s if count < 1000
608
+
609
+ count < 999_950 ? format('%.1fk', count / 1e3) : format('%.1fM', count / 1e6)
610
+ end
611
+
612
+ module_function def format_cost(dollars)
613
+ dollars < 0.01 ? '<$0.01' : format('$%.2f', dollars)
614
+ end
615
+
616
+ # The sum of usage hashes (see HARNESS), ignoring nils (and anything else
617
+ # not a hash); nil if there are none.
618
+ # The cost is summed only if every usage has one, so a partial total
619
+ # isn't presented as the whole.
620
+ module_function def sum_usage(usages)
621
+ usages = usages.grep(Hash)
622
+ return nil if usages.empty?
623
+
624
+ total = %w[input output cache_read cache_write].to_h { |key| [key, usages.sum { |u| u[key].to_i }] }
625
+ costs = usages.map { |u| u['cost'] }
626
+ costs.all?(Numeric) ? total.merge('cost' => costs.sum.round(6)) : total
627
+ end
628
+
629
+ # The run dir's machine-readable record of its agents' runs.
630
+ RUN_FILENAME = 'run.json'
631
+
632
+ # A run.json entry for an agent's run_agent result and its elapsed
633
+ # seconds: its report's path under dir (the run dir's reviews_dir for
634
+ # analysts) on success, its error on failure. The harness is per entry,
635
+ # not per run, so a run can mix harnesses. meta_report: for a
636
+ # consolidation whose report was split (its detail is the findings file's
637
+ # name), the other file, which carries the front matter.
638
+ module_function def run_entry(agent, ok, detail, elapsed, usage, index: nil, dir: nil, meta_report: nil)
639
+ entry = { 'agent_index' => index, 'harness' => HARNESS::NAME, 'provider' => agent['provider'],
640
+ 'model' => agent['model'], 'ok' => ok }.compact
641
+ entry[ok ? 'report' : 'error'] = ok && dir ? File.join(dir, detail) : detail
642
+ entry['meta_report'] = (dir ? File.join(dir, meta_report) : meta_report) if ok && meta_report
643
+ entry.merge('duration_seconds' => elapsed.round(1), 'usage' => usage)
644
+ end
645
+
646
+ # Writes the run dir's run.json: every analyst's and consolidation's
647
+ # entry (see run_entry), their total usage, and how many runs it covers
648
+ # (those that reported usage). Replaced atomically, so a
649
+ # reader or a Ctrl-C never sees it half written. A failure to write it is
650
+ # only warned about: the reports are what matter.
651
+ module_function def write_run_file(run_dir, analysts, consolidations)
652
+ usages = (analysts + consolidations).map { |entry| entry['usage'] }
653
+ data = { 'analysts' => analysts, 'consolidations' => consolidations,
654
+ 'total_usage' => sum_usage(usages), 'total_usage_runs' => usages.grep(Hash).size }
655
+ path = File.join(run_dir, RUN_FILENAME)
656
+ temp = "#{path}.#{Process.pid}.tmp"
657
+ File.write(temp, "#{JSON.pretty_generate(data)}\n")
658
+ File.rename(temp, path)
659
+ rescue SystemCallError => e
660
+ FileUtils.rm_f(temp) if temp
661
+ warn "Could not write #{relative_path(path)}: #{e.message}"
662
+ end
663
+
664
+ # [analysts, consolidations] from a run dir's run.json, or nil if it is
665
+ # missing or malformed.
666
+ module_function def read_run_file(run_dir)
667
+ data = JSON.parse(File.read(File.join(run_dir, RUN_FILENAME), encoding: ENCODING))
668
+ lists = data.values_at('analysts', 'consolidations') if data.is_a?(Hash)
669
+ lists if lists&.all?(Array)
670
+ rescue SystemCallError, JSON::ParserError
671
+ nil
672
+ end
673
+
674
+ # Runs the block; returns [block_result, elapsed_seconds] (monotonic clock).
675
+ module_function def timed
676
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
677
+ result = yield
678
+ [result, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started]
679
+ end
680
+
681
+ RESULT_HEADINGS = %w[Ok Duration Agent In Cached Out Cost].freeze
682
+ # The usage columns, right-aligned so their figures line up.
683
+ RIGHT_ALIGNED_COLUMNS = (3..6)
684
+
685
+ # Indented result table lines for [label, ok, detail, elapsed, usage]
686
+ # results: a heading row, then one row per result, with ✓ or ✗ and its
687
+ # usage (as for usage_cells), then the failures' errors, if any, listed
688
+ # below the table so long messages don't widen it. Report filenames
689
+ # aren't shown: an analyst's is found by its agent number, and the
690
+ # consolidated report is at the run dir's top level.
691
+ module_function def format_result_table(results)
692
+ rows = results.map do |label, ok, _detail, elapsed, usage|
693
+ [ok ? '✓' : '✗', format_duration(elapsed), label, *usage_cells(usage)]
694
+ end
695
+ table = [RESULT_HEADINGS, *rows]
696
+ widths = table.transpose.map { |column| column.map { |cell| display_width(cell) }.max }
697
+ lines = table.map do |row|
698
+ cells = row.each_with_index.map do |cell, i|
699
+ padding = ' ' * (widths[i] - display_width(cell))
700
+ RIGHT_ALIGNED_COLUMNS.cover?(i) ? padding + cell : cell + padding
701
+ end
702
+ " #{cells.join(' ')}".rstrip
703
+ end
704
+ errors = results.reject { |_, ok| ok }.map { |label, _, detail| " #{label}: #{detail}" }
705
+ errors.empty? ? lines : [*lines, '', ' Errors:', *errors]
706
+ end
707
+
708
+ # "; <total usage>" for a summary line, noting how many of the runs it
709
+ # covers when some reported no usage; "" when none reported any.
710
+ module_function def usage_summary(usages)
711
+ total = format_usage(sum_usage(usages))
712
+ return '' unless total
713
+
714
+ reported = usages.grep(Hash).size
715
+ "; #{total}#{" (from #{reported} of #{usages.size} agents)" if reported < usages.size}"
716
+ end
717
+
718
+ # East Asian wide and fullwidth characters (CJK, Hangul, fullwidth forms,
719
+ # most emoji), which terminals show two columns wide.
720
+ WIDE_CHAR = Regexp.new(
721
+ '[\u1100-\u115F\u2E80-\u303E\u3041-\u33FF\u3400-\u4DBF\u4E00-\u9FFF' \
722
+ '\uA000-\uA4CF\uAC00-\uD7A3\uF900-\uFAFF\uFE30-\uFE4F\uFF00-\uFF60' \
723
+ '\uFFE0-\uFFE6\u{1F300}-\u{1F64F}\u{1F900}-\u{1F9FF}\u{20000}-\u{3FFFD}]'
724
+ )
725
+ # Combining marks and zero-width characters, which take no column.
726
+ ZERO_WIDTH_CHAR = /[\p{Mn}\p{Me}\u200B-\u200F\uFEFF]/
727
+
728
+ # The number of terminal columns text takes: as String#length, but
729
+ # counting wide characters as two and zero-width ones as none.
730
+ # Approximate (e.g. ambiguous-width characters count as one), but enough
731
+ # to keep result tables aligned.
732
+ module_function def display_width(text)
733
+ text.length + text.scan(WIDE_CHAR).size - text.scan(ZERO_WIDTH_CHAR).size
734
+ end
735
+
736
+ # Prints the "--- Consolidation complete ---" (or error) marker.
737
+ module_function def puts_consolidation_status(ok)
738
+ puts "\n--- Consolidation #{ok ? 'complete' : 'error'} ---"
739
+ end
740
+
741
+ # True if a config entry (agent or consolidator) has the required string
742
+ # 'provider' and 'model' fields.
743
+ module_function def valid_agent_entry?(entry)
744
+ entry.is_a?(Hash) && entry['provider'].is_a?(String) && entry['model'].is_a?(String)
745
+ end
746
+
747
+ # True if `name` (a bare command name) is an executable file in a PATH
748
+ # directory. Checked up front so a missing harness fails before any run
749
+ # dir is created, rather than as an Errno::ENOENT from every agent. An
750
+ # empty PATH entry means the current directory, as it does for execvp (-1
751
+ # keeps a trailing empty entry).
752
+ module_function def executable_on_path?(name)
753
+ ENV.fetch('PATH', '').split(File::PATH_SEPARATOR, -1).map { |dir| dir.empty? ? '.' : dir }.any? do |dir|
754
+ path = File.join(dir, name)
755
+ File.executable?(path) && !File.directory?(path)
756
+ end
757
+ end
758
+
759
+ # "harness / provider / model" for a validated config entry, printed at
760
+ # startup so it is clear which agents are still running later.
761
+ module_function def agent_descriptor(entry)
762
+ "#{HARNESS::NAME} / #{entry['provider']} / #{entry['model']}"
763
+ end
764
+
765
+ # AGENT_COUNCIL_OPTS as UTF-8, or nil when unset. Retagged because ENV
766
+ # values carry the locale's encoding (binary under LANG=C), which would let
767
+ # invalid bytes through unchecked.
768
+ module_function def env_options_value(env = ENV)
769
+ env[ENV_OPTS_VAR]&.dup&.force_encoding(Encoding::UTF_8)
770
+ end
771
+
772
+ # Shell-split tokens from AGENT_COUNCIL_OPTS, or [] when unset.
773
+ module_function def env_options(env = ENV)
774
+ raw = env_options_value(env)
775
+ return [] if raw.nil? || raw.strip.empty?
776
+
777
+ Shellwords.shellsplit(raw)
778
+ rescue ArgumentError, EncodingError => e # e.g. an unclosed quote, invalid UTF-8
779
+ raise ArgumentError, "Invalid #{ENV_OPTS_VAR} value: #{e.message}"
780
+ end
781
+
782
+ # options: receives the parsed option values (:consolidate, :output_dir).
783
+ # env: where -h finds the current AGENT_COUNCIL_OPTS value.
784
+ module_function def option_parser(options = {}, env = ENV)
785
+ OptionParser.new do |opts|
786
+ opts.banner = <<~BANNER
787
+ -------------------------------------------------------------------------------
788
+ Usage: agent-council [options] PROMPT_FILE [CONFIG_FILE]
789
+ agent-council --consolidate RUN_DIR
790
+ Repository: https://github.com/keithrbennett/agent-council
791
+ Documentation (Local): #{File.join(PROJECT_ROOT, 'README.md')}
792
+ Examples (Local): #{EXAMPLES_DIR}
793
+ Version: #{VERSION}
794
+ -------------------------------------------------------------------------------
795
+
796
+ Arguments:
797
+ PROMPT_FILE Path to the prompt file to send to every configured model (required)
798
+ CONFIG_FILE Path to a YAML agent config (default: ./#{CWD_CONFIG_NAME}, else ~/#{HOME_CONFIG_NAME}; required if neither exists)
799
+ RUN_DIR With --consolidate, a finished run's directory (no PROMPT_FILE or CONFIG_FILE)
800
+
801
+ Agents analyze the current directory: run from the project to be analyzed.
802
+
803
+ Options:
804
+ BANNER
805
+
806
+ opts.on('--consolidate RUN_DIR',
807
+ 'Consolidate an existing run\'s reports, in place; runs no analysts') do |dir|
808
+ options[:consolidate] = dir
809
+ end
810
+
811
+ opts.on('-o', '--output-dir DIR',
812
+ 'Base directory for run dirs, as DIR/<date>/<time>',
813
+ "(default: ./#{DEFAULT_REPORTS_DIR_NAME})",
814
+ 'Not allowed with --consolidate') do |dir|
815
+ options[:output_dir] = dir
816
+ end
817
+
818
+ opts.on('-h', '--help', 'Show this help and exit') do
819
+ puts opts
820
+ exit 0
821
+ end
822
+
823
+ opts.on('-v', '--version', 'Show the version and exit') do
824
+ puts "agent-council #{VERSION}"
825
+ exit 0
826
+ end
827
+
828
+ opts.separator ''
829
+ opts.separator 'Environment:'
830
+ opts.separator " #{ENV_OPTS_VAR} holds arguments inserted before the command-line ones,"
831
+ opts.separator ' so an option given on the command line overrides the same option there.'
832
+ opts.separator " Example: export #{ENV_OPTS_VAR}=\"--output-dir tmp/council\""
833
+ # Inspected, so newlines can't break -h. (A value with invalid bytes
834
+ # never gets here: env_options rejects it first.)
835
+ env_value = env_options_value(env).to_s
836
+ opts.separator " Currently set to: #{env_value.inspect}" unless env_value.strip.empty?
837
+ opts.separator ''
838
+ opts.separator 'Examples:'
839
+ opts.separator ' agent-council my-analysis-prompt.md'
840
+ opts.separator ' agent-council my-analysis-prompt.md other-config.yml'
841
+ opts.separator ' agent-council -o tmp/council my-analysis-prompt.md'
842
+ opts.separator " agent-council --consolidate #{DEFAULT_REPORTS_DIR_NAME}/2026-09-03/09-14-02"
843
+ end
844
+ end
845
+
846
+ module_function def main(argv)
847
+ abort_if_native_windows
848
+ options = {}
849
+ parser = option_parser(options)
850
+ begin
851
+ argv = parser.parse(env_options + argv)
852
+ rescue ArgumentError => e # a malformed AGENT_COUNCIL_OPTS
853
+ abort e.message
854
+ rescue OptionParser::ParseError => e
855
+ warn e.message
856
+ warn parser
857
+ exit 1
858
+ end
859
+
860
+ if options[:consolidate]
861
+ unless argv.empty?
862
+ warn parser
863
+ exit 1
864
+ end
865
+ # The consolidation report goes into RUN_DIR itself.
866
+ if options.key?(:output_dir)
867
+ abort "--output-dir (given directly or in #{ENV_OPTS_VAR}) cannot be used with --consolidate, " \
868
+ 'which writes into RUN_DIR'
869
+ end
870
+ consolidate_run(options[:consolidate])
871
+ end
872
+
873
+ # Checked up front, so a bad value fails before any work, with a message
874
+ # rather than a backtrace. A blank value (e.g. -o "$UNSET_VAR") would
875
+ # otherwise resolve to the cwd and scatter day dirs over the project.
876
+ if options[:output_dir]&.strip&.empty?
877
+ abort '--output-dir needs a directory; got an empty value'
878
+ end
879
+ reports_root =
880
+ begin
881
+ reports_dir(options[:output_dir])
882
+ rescue ArgumentError => e # e.g. ~nosuchuser/x
883
+ abort "Invalid --output-dir #{options[:output_dir]}: #{e.message}"
884
+ end
885
+ # mkdir_p would fail on a file anywhere along the path, so check the
886
+ # nearest part that exists, which is the dir itself if it exists.
887
+ existing = reports_root
888
+ existing = File.dirname(existing) until File.exist?(existing)
889
+ unless File.directory?(existing)
890
+ abort "Output dir #{relative_path(reports_root)} can't be created: " \
891
+ "#{relative_path(existing)} is not a directory"
892
+ end
893
+
894
+ prompt_path = argv[0]
895
+
896
+ if prompt_path.nil? || argv.length > 2
897
+ warn parser
898
+ exit 1
899
+ end
900
+ abort "Prompt file not found: #{prompt_path}" unless File.file?(prompt_path)
901
+
902
+ config_path = resolve_config_path(argv[1])
903
+ unless config_path
904
+ abort <<~MESSAGE
905
+ No config file given, and neither ./#{CWD_CONFIG_NAME} nor ~/#{HOME_CONFIG_NAME} exists.
906
+ Pass a config path: agent-council PROMPT_FILE CONFIG_FILE
907
+ Minimal config: agents: [{provider: ..., model: ...}]
908
+ Sample configs: #{File.join(EXAMPLES_DIR, 'configs')}
909
+ MESSAGE
910
+ end
911
+ abort "Config file not found: #{config_path}" unless File.file?(config_path)
912
+
913
+ # Read once. Analysts receive the user's prompt followed by the
914
+ # report-format addendum (see compose_analyst_prompt); the consolidator
915
+ # cites the prompt alone.
916
+ # The prompt and analyst_format.md are copied into the run dir, so the
917
+ # exact bytes sent to each analyst are recoverable.
918
+ prompt = File.read(prompt_path, encoding: ENCODING)
919
+ abort "Prompt file is empty: #{prompt_path}" if prompt.empty?
920
+ abort "Analyst format addendum missing: #{ANALYST_FORMAT_PATH}" unless File.file?(ANALYST_FORMAT_PATH)
921
+ # All three are copied into the run dir under their own names, so a
922
+ # shared name would leave one copy in place of another.
923
+ copied_paths = { 'prompt' => prompt_path, 'config' => config_path, 'addendum' => ANALYST_FORMAT_PATH }
924
+ copied_names = copied_paths.transform_values { |path| File.basename(path) }
925
+ unless copied_names.values.uniq.size == copied_names.size
926
+ abort "The prompt, config, and #{File.basename(ANALYST_FORMAT_PATH)} need different file names, " \
927
+ 'since all three are copied into the run dir: ' \
928
+ "#{copied_names.map { |role, name| "#{role} #{name}" }.join(', ')}"
929
+ end
930
+ if (role = copied_names.key(REVIEWS_DIR_NAME))
931
+ abort "The #{role} can't be named #{REVIEWS_DIR_NAME}, the run dir's subdirectory for analyst reports"
932
+ end
933
+ analyst_prompt = compose_analyst_prompt(prompt, File.read(ANALYST_FORMAT_PATH, encoding: ENCODING))
934
+
935
+ config =
936
+ begin
937
+ YAML.safe_load_file(config_path)
938
+ rescue Psych::Exception => e
939
+ abort "Invalid YAML in #{config_path}: #{e.message}"
940
+ end
941
+ abort "Config file must be a YAML mapping: #{config_path}" unless config.is_a?(Hash)
942
+
943
+ agents = config['agents']
944
+ abort "No agents defined in #{config_path}" unless agents.is_a?(Array) && !agents.empty?
945
+ unless agents.all? { |a| valid_agent_entry?(a) }
946
+ abort "Each agent in #{config_path} needs string 'provider' and 'model' fields"
947
+ end
948
+
949
+ consolidator = config['consolidator']
950
+ if consolidator
951
+ unless valid_agent_entry?(consolidator)
952
+ abort "Consolidator in #{config_path} needs string 'provider' and 'model' fields"
953
+ end
954
+ unless File.file?(CONSOLIDATE_PROMPT_PATH)
955
+ abort "Consolidator configured but template missing: #{CONSOLIDATE_PROMPT_PATH}"
956
+ end
957
+ end
958
+
959
+ abort_unless_harness_installed
960
+
961
+ # One run dir per invocation, day-bucketed so the reports dir stays
962
+ # navigable at volume: <reports dir>/<yyyy-mm-dd>/<hh-mm-ss>/. The dir name is pure time
963
+ # (plus a collision suffix, see claim_run_dir); full timestamp and query
964
+ # slug go into the report filenames (file_prefix).
965
+ now = Time.now
966
+ day = now.strftime('%Y-%m-%d')
967
+ day_dir = File.join(reports_root, day)
968
+ run_name =
969
+ begin
970
+ claim_run_dir(day_dir, now.strftime('%H-%M-%S'))
971
+ rescue SystemCallError => e # e.g. no write permission
972
+ abort "Cannot create a run dir under #{relative_path(reports_root)}: #{e.message}"
973
+ end
974
+ run_stamp = "#{day}-#{run_name}"
975
+ iso_stamp = iso_timestamp(now)
976
+ slug = query_slug(prompt_path)
977
+ file_prefix = "#{run_stamp}_#{slug}"
978
+ # prompt_file and config_file name the input copies in the run dir, and
979
+ # run_stamp gives the report filenames' prefix, so --consolidate needn't
980
+ # rely on the run dir's path.
981
+ report_meta = { 'role' => 'analyst', 'timestamp' => iso_stamp, 'query' => slug,
982
+ 'prompt_file' => File.basename(prompt_path), 'config_file' => File.basename(config_path),
983
+ 'config_source' => display_source_path(config_path), 'run_stamp' => run_stamp }
984
+ run_dir = File.join(day_dir, run_name)
985
+ reviews_dir = File.join(run_dir, REVIEWS_DIR_NAME)
986
+ FileUtils.mkdir_p(reviews_dir)
987
+
988
+ # Keep the run's inputs recoverable: copy the prompt, format
989
+ # addendum, and config. (The orchestrator is not copied: it is
990
+ # versioned in git.)
991
+ FileUtils.cp(prompt_path, run_dir)
992
+ FileUtils.cp(ANALYST_FORMAT_PATH, run_dir)
993
+ FileUtils.cp(config_path, run_dir)
994
+
995
+ # A default config's path is absolute (resolve_config_path makes it
996
+ # so), to show which of the defaults was picked up.
997
+ puts
998
+ puts(argv[1] ? "Config: #{relative_path(config_path)}" : "Config: [default] #{config_path}")
999
+ puts "Prompt: #{relative_path(prompt_path)}"
1000
+ puts "Run Dir: #{relative_path(run_dir)}"
1001
+ puts "\nReview Agents (#{agents.size}):"
1002
+ agents.each_with_index do |agent, i|
1003
+ puts " ##{i + 1} #{agent_descriptor(agent)}"
1004
+ end
1005
+ puts "\nConsolidator Agent:\n #{agent_descriptor(consolidator)}" if consolidator
1006
+ puts "\nReview Progress:"
1007
+ pool = WorkerPool.new(grace: INTERRUPT_GRACE_SECONDS)
1008
+ pool.trap_signals
1009
+ agents.each_with_index do |agent, i|
1010
+ index = i + 1
1011
+ pool.start do
1012
+ provider = agent.is_a?(Hash) ? agent['provider'] : nil
1013
+ model = agent.is_a?(Hash) ? agent['model'] : nil
1014
+ meta = report_meta.merge('agent_index' => index)
1015
+ (label, ok, detail, usage), elapsed = timed do
1016
+ run_agent(agent, analyst_prompt, reviews_dir, file_prefix,
1017
+ index: index,
1018
+ meta: meta,
1019
+ header: ->(run_fields) { byline_table(meta.merge(run_fields), provider, model) },
1020
+ pool: pool)
1021
+ end
1022
+ [label, ok, detail, elapsed, usage]
1023
+ end
1024
+ end
1025
+ results = pool.collect do |label, ok, detail, elapsed|
1026
+ duration = format_duration(elapsed)
1027
+ puts(ok ? " ✓ #{duration} #{label}" : " ✗ #{duration} #{label}\n #{detail}")
1028
+ end
1029
+
1030
+ failed = results.count { |_, ok, _| !ok }
1031
+ puts "\nReviews (in #{relative_path(reviews_dir)}):"
1032
+ puts format_result_table(results)
1033
+ puts "\n--- Reviews complete (#{results.size - failed}/#{results.size} succeeded" \
1034
+ "#{usage_summary(results.map(&:last))}) ---"
1035
+ # Written now, so it's there even if the consolidation is interrupted.
1036
+ analyst_runs = agents.zip(results).each_with_index.map do |(agent, (_, ok, detail, elapsed, usage)), i|
1037
+ run_entry(agent, ok, detail, elapsed, usage, index: i + 1, dir: REVIEWS_DIR_NAME)
1038
+ end
1039
+ write_run_file(run_dir, analyst_runs, [])
1040
+
1041
+ if consolidator
1042
+ puts "\nConsolidation Report (in #{relative_path(run_dir)}):"
1043
+ failures = results.reject { |_, ok, _| ok }.map { |label, _, detail, _| [label, detail] }
1044
+ pool.start do
1045
+ timed do
1046
+ run_consolidation(consolidator, prompt, run_dir, file_prefix,
1047
+ meta: report_meta, prompt_path: prompt_path, failures: failures, analyst_runs: analyst_runs,
1048
+ pool: pool)
1049
+ end
1050
+ end
1051
+ (label, ok, detail, usage, meta_name), elapsed = pool.collect.first
1052
+ puts format_result_table([[label, ok, detail, elapsed, usage]])
1053
+ entry = run_entry(consolidator, ok, detail, elapsed, usage, meta_report: meta_name)
1054
+ write_run_file(run_dir, analyst_runs, [entry])
1055
+ puts_consolidation_status(ok)
1056
+ end
1057
+
1058
+ exit(failed.zero? ? 0 : 1)
1059
+ rescue Interrupt
1060
+ exit_interrupted(pool)
1061
+ end
1062
+
1063
+ # --consolidate RUN_DIR: runs the consolidator over an existing
1064
+ # run's reports, writing its report into that run dir. The prompt,
1065
+ # config, and run's identity (timestamp, query, run_stamp) all come from
1066
+ # the run itself: its analysts' front matter names the prompt and config
1067
+ # copies.
1068
+ # The consolidator is the config copy's; edit that copy to try another.
1069
+ # Exits 0 only if the consolidation succeeded.
1070
+ module_function def consolidate_run(run_dir)
1071
+ abort "Run dir not found: #{run_dir}" unless File.directory?(run_dir)
1072
+ reviews_dir = File.join(run_dir, REVIEWS_DIR_NAME)
1073
+ reports = File.directory?(reviews_dir) ? collect_reports(reviews_dir) : []
1074
+ abort "No analyst reports found in #{reviews_dir}" if reports.empty?
1075
+
1076
+ # From the first report that has them, so one stray or older report in
1077
+ # reviews/ doesn't block the run.
1078
+ fields = reports.map { |report| report[2] }
1079
+ .find { |f| f.values_at('prompt_file', 'config_file', 'run_stamp', 'query').all? }
1080
+ unless fields
1081
+ abort "Reports in #{reviews_dir} lack prompt_file, config_file, run_stamp, or query; " \
1082
+ 'runs made before --consolidate existed cannot be consolidated'
1083
+ end
1084
+ %w[prompt_file config_file].each do |key|
1085
+ abort "Invalid #{key} in #{reviews_dir}: #{fields[key].inspect}" unless bare_name?(fields[key])
1086
+ end
1087
+ prompt_path = File.join(run_dir, fields['prompt_file'])
1088
+ config_path = File.join(run_dir, fields['config_file'])
1089
+ abort "Prompt copy not found: #{prompt_path}" unless File.file?(prompt_path)
1090
+ abort "Config copy not found: #{config_path}" unless File.file?(config_path)
1091
+
1092
+ config =
1093
+ begin
1094
+ YAML.safe_load_file(config_path)
1095
+ rescue Psych::Exception => e
1096
+ abort "Invalid YAML in #{config_path}: #{e.message}"
1097
+ end
1098
+ consolidator = config['consolidator'] if config.is_a?(Hash)
1099
+ abort "No consolidator in #{config_path}; add one to consolidate" unless consolidator
1100
+ unless valid_agent_entry?(consolidator)
1101
+ abort "Consolidator in #{config_path} needs string 'provider' and 'model' fields"
1102
+ end
1103
+ unless File.file?(CONSOLIDATE_PROMPT_PATH)
1104
+ abort "Consolidation template missing: #{CONSOLIDATE_PROMPT_PATH}"
1105
+ end
1106
+
1107
+ file_prefix = "#{fields['run_stamp']}_#{fields['query']}" # as in a normal run
1108
+ filename = consolidated_filename(file_prefix, consolidator['provider'], consolidator['model'])
1109
+ unless bare_name?(filename)
1110
+ abort "Invalid run_stamp or query in #{reviews_dir}: #{fields.values_at('run_stamp', 'query').inspect}"
1111
+ end
1112
+ names = [nil, 'findings', 'meta'].map do |part|
1113
+ consolidated_filename(file_prefix, consolidator['provider'], consolidator['model'], part)
1114
+ end
1115
+ existing = names.select { |name| File.exist?(File.join(run_dir, name)) }
1116
+ unless existing.empty?
1117
+ listed = existing.map { |name| relative_path(File.join(run_dir, name)) }.join(', ')
1118
+ abort "Consolidated report already exists: #{listed}\n" \
1119
+ "Rename or delete #{existing.size == 1 ? 'it' : 'all of them'} to consolidate again with the " \
1120
+ 'same model configuration.'
1121
+ end
1122
+ prompt = File.read(prompt_path, encoding: ENCODING)
1123
+ abort "Prompt copy is empty: #{prompt_path}" if prompt.empty?
1124
+
1125
+ abort_unless_harness_installed
1126
+
1127
+ # In the same key order as a normal run's front matter.
1128
+ meta = fields.slice('role', 'timestamp', 'query', 'prompt_file', 'config_file', 'config_source',
1129
+ 'run_stamp')
1130
+ meta['timestamp'] = iso_timestamp(meta['timestamp']) if meta['timestamp'].is_a?(Time)
1131
+
1132
+ puts "Config: #{relative_path(config_path)}"
1133
+ puts "Prompt: #{relative_path(prompt_path)}"
1134
+ puts "Run Dir: #{relative_path(run_dir)}"
1135
+ puts "\nConsolidator Agent:\n #{agent_descriptor(consolidator)}"
1136
+ puts "\nConsolidation Report (in #{relative_path(run_dir)}, " \
1137
+ "from #{reports.size} review#{'s' unless reports.size == 1}):"
1138
+ failures = missing_agents(config, reports)
1139
+ analysts, consolidations = read_run_file(run_dir)
1140
+ pool = WorkerPool.new(grace: INTERRUPT_GRACE_SECONDS)
1141
+ pool.trap_signals
1142
+ pool.start do
1143
+ timed do
1144
+ run_consolidation(consolidator, prompt, run_dir, file_prefix,
1145
+ meta: meta, prompt_path: relative_path(prompt_path), failures: failures,
1146
+ analyst_runs: analysts || [], pool: pool)
1147
+ end
1148
+ end
1149
+ (label, ok, detail, usage, meta_name), elapsed = pool.collect.first
1150
+ puts format_result_table([[label, ok, detail, elapsed, usage]])
1151
+ # Added to the run's run.json. Without one (deleted, or unreadable),
1152
+ # none is written: a total covering only this consolidation would
1153
+ # pass for the run's.
1154
+ if analysts
1155
+ entry = run_entry(consolidator, ok, detail, elapsed, usage, meta_report: meta_name)
1156
+ write_run_file(run_dir, analysts, consolidations + [entry])
1157
+ else
1158
+ warn "No readable #{RUN_FILENAME} in #{relative_path(run_dir)}; not recording this consolidation's run"
1159
+ end
1160
+ puts_consolidation_status(ok)
1161
+ exit(ok ? 0 : 1)
1162
+ rescue Interrupt
1163
+ exit_interrupted(pool)
1164
+ end
1165
+
1166
+ # True if name is a plain file name: no directory part or null byte, and
1167
+ # not empty, "." or "..".
1168
+ # Applied to every front matter value that becomes part of a path, so a
1169
+ # crafted report can't make --consolidate read or write outside the run
1170
+ # dir.
1171
+ module_function def bare_name?(name)
1172
+ name.is_a?(String) && !name.include?("\0") && !['', '.', '..'].include?(name) &&
1173
+ File.basename(name) == name
1174
+ end
1175
+
1176
+ # What --consolidate tells the consolidator about an agent without a report.
1177
+ MISSING_REPORT_REASON =
1178
+ 'reason not recorded (it may have failed, been interrupted, or had its report removed)'
1179
+
1180
+ # For --consolidate: the config copy's agents that have no report, as
1181
+ # [label, reason] pairs. Why wasn't recorded, and needn't be a failure:
1182
+ # the run may have been interrupted, or a report removed.
1183
+ module_function def missing_agents(config, reports)
1184
+ agents = config['agents'].is_a?(Array) ? config['agents'] : []
1185
+ present = reports.map(&:first)
1186
+ agents.each_with_index.filter_map do |agent, i|
1187
+ next unless valid_agent_entry?(agent)
1188
+
1189
+ label = agent_label(agent['provider'], agent['model'], i + 1)
1190
+ [label, MISSING_REPORT_REASON] unless present.include?(label)
1191
+ end
1192
+ end
1193
+
1194
+ # Checked up front, before any run dir is created or model called.
1195
+ module_function def abort_unless_harness_installed
1196
+ return if executable_on_path?(HARNESS::NAME)
1197
+
1198
+ abort <<~MESSAGE
1199
+ #{HARNESS::NAME} executable not found; is `#{HARNESS::NAME}` installed and on your PATH?
1200
+ See "Prerequisites" in #{File.join(PROJECT_ROOT, 'README.md')}
1201
+ MESSAGE
1202
+ end
1203
+
1204
+ # After Ctrl-C (or another signal WorkerPool#trap_signals traps):
1205
+ # stops pool's agents, if any, and dies by the signal.
1206
+ module_function def exit_interrupted(pool)
1207
+ report = -> do
1208
+ warn "\nInterrupted."
1209
+ rescue IOError, SystemCallError
1210
+ nil # e.g. the terminal hung up
1211
+ end
1212
+ if pool
1213
+ pool.shutdown(&report)
1214
+ pool.exit_by_signal
1215
+ else # before any agent started; only a Ctrl-C can get here
1216
+ report.call
1217
+ WorkerPool.exit_by_signal('INT')
1218
+ end
1219
+ end
1220
+ end