agent-council 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/DESIGN.md +378 -0
- data/INSTALL.md +98 -0
- data/LICENSE.txt +21 -0
- data/README.md +852 -0
- data/RELEASE_NOTES.md +25 -0
- data/code-review-agents.md +28 -0
- data/examples/configs/consolidated.yml +22 -0
- data/examples/configs/single.yml +10 -0
- data/examples/prompts/dnsruby-pull-221-prompt.md +4 -0
- data/examples/prompts/project-name-prompt.md +3 -0
- data/examples/prompts/recommend-model-ensemble-prompt.md +62 -0
- data/examples/prompts/review-docs-prompt.md +101 -0
- data/examples/prompts/review-latest-commit-prompt.md +46 -0
- data/examples/prompts/review-pull-request-prompt.md +91 -0
- data/examples/prompts/smoke-test-prompt.md +1 -0
- data/examples/prompts/state-of-the-code-base-prompt.md +172 -0
- data/examples/prompts/test-coverage-analysis-prompt.md +346 -0
- data/exe/agent-council +12 -0
- data/exe/pi-models +39 -0
- data/lib/agent_council/harnesses/pi_harness.rb +169 -0
- data/lib/agent_council/pi_models.rb +185 -0
- data/lib/agent_council/platform.rb +27 -0
- data/lib/agent_council/templates/analyst_format.md +12 -0
- data/lib/agent_council/templates/consolidate_prompt.md +72 -0
- data/lib/agent_council/version.rb +5 -0
- data/lib/agent_council/worker_pool.rb +217 -0
- data/lib/agent_council.rb +1220 -0
- metadata +84 -0
|
@@ -0,0 +1,1220 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# AgentCouncil — harness-agnostic orchestration: naming, run-directory
|
|
4
|
+
# layout, per-agent subprocess execution, and the CLI entry point.
|
|
5
|
+
# Harness-specific behavior lives under AgentCouncil::Harnesses (e.g.
|
|
6
|
+
# AgentCouncil::Harnesses::PiHarness); HARNESS below is the single swap
|
|
7
|
+
# point.
|
|
8
|
+
|
|
9
|
+
require 'json'
|
|
10
|
+
require 'yaml'
|
|
11
|
+
# Psych requires stringio lazily, on first emit. On JRuby, worker threads
|
|
12
|
+
# emitting front matter at once can then see a half-loaded StringIO
|
|
13
|
+
# (ArgumentError from StringIO.new), so load it before any threads start.
|
|
14
|
+
require 'stringio'
|
|
15
|
+
require 'open3'
|
|
16
|
+
require 'fileutils'
|
|
17
|
+
require 'tmpdir'
|
|
18
|
+
require 'optparse'
|
|
19
|
+
require 'shellwords'
|
|
20
|
+
|
|
21
|
+
require_relative 'agent_council/version'
|
|
22
|
+
require_relative 'agent_council/platform'
|
|
23
|
+
require_relative 'agent_council/harnesses/pi_harness'
|
|
24
|
+
require_relative 'agent_council/worker_pool'
|
|
25
|
+
|
|
26
|
+
module AgentCouncil
|
|
27
|
+
# Runtime data files (prompt templates) ship inside lib/, resolved
|
|
28
|
+
# relative to this file, so the gem is self-contained: what gets loaded at runtime never depends on which
|
|
29
|
+
# files happen to survive the gemspec's file list. PROJECT_ROOT remains
|
|
30
|
+
# only for documentation references (e.g. the local README path in help
|
|
31
|
+
# text), not for runtime data.
|
|
32
|
+
PROJECT_ROOT = File.expand_path('..', __dir__)
|
|
33
|
+
EXAMPLES_DIR = File.join(PROJECT_ROOT, 'examples')
|
|
34
|
+
TEMPLATES_DIR = File.join(__dir__, 'agent_council', 'templates')
|
|
35
|
+
CONSOLIDATE_PROMPT_PATH = File.join(TEMPLATES_DIR, 'consolidate_prompt.md')
|
|
36
|
+
ANALYST_FORMAT_PATH = File.join(TEMPLATES_DIR, 'analyst_format.md')
|
|
37
|
+
|
|
38
|
+
# The harness every model runs through; the single swap point. A second
|
|
39
|
+
# harness would be another module under AgentCouncil::Harnesses,
|
|
40
|
+
# assigned here. Its usage(stdout) returns token counts in the shape
|
|
41
|
+
# documented on PiHarness.usage, or nil when the harness reports none.
|
|
42
|
+
HARNESS = AgentCouncil::Harnesses::PiHarness
|
|
43
|
+
|
|
44
|
+
# After Ctrl-C, how long agents get to exit after SIGTERM before they're
|
|
45
|
+
# killed (see WorkerPool).
|
|
46
|
+
INTERRUPT_GRACE_SECONDS = 5
|
|
47
|
+
|
|
48
|
+
# All text read or captured (prompts, templates, reports, harness output)
|
|
49
|
+
# is UTF-8, whatever the locale's default external encoding (US-ASCII
|
|
50
|
+
# under LANG=C or with LANG unset).
|
|
51
|
+
ENCODING = Encoding::UTF_8
|
|
52
|
+
|
|
53
|
+
# By default, reports go under the current working directory — i.e.
|
|
54
|
+
# alongside the repository being analyzed — so the tool can be run from
|
|
55
|
+
# any project. The name says which tool wrote them, since they land in
|
|
56
|
+
# someone else's tree.
|
|
57
|
+
DEFAULT_REPORTS_DIR_NAME = 'agent-council-reports'
|
|
58
|
+
|
|
59
|
+
# The reports root: the --output-dir value if given (relative paths
|
|
60
|
+
# resolve against the cwd), else ./agent-council-reports.
|
|
61
|
+
module_function def reports_dir(explicit = nil)
|
|
62
|
+
File.expand_path(explicit || DEFAULT_REPORTS_DIR_NAME)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Config file names picked up when no CONFIG_FILE argument is given: a
|
|
66
|
+
# visible per-project file in the current working directory, and a hidden
|
|
67
|
+
# (dotfile) per-user file in the home directory.
|
|
68
|
+
CWD_CONFIG_NAME = 'agent-council.yml'
|
|
69
|
+
HOME_CONFIG_NAME = '.agent-council.yml'
|
|
70
|
+
|
|
71
|
+
# Shell-style arguments from the environment, inserted before ARGV.
|
|
72
|
+
ENV_OPTS_VAR = 'AGENT_COUNCIL_OPTS'
|
|
73
|
+
|
|
74
|
+
# The explicit CONFIG_FILE argument if given, else ./agent-council.yml if
|
|
75
|
+
# it exists, else ~/.agent-council.yml if it exists; nil when none apply.
|
|
76
|
+
# Only the cwd is checked (no upward walk).
|
|
77
|
+
module_function def resolve_config_path(explicit)
|
|
78
|
+
return explicit if explicit
|
|
79
|
+
|
|
80
|
+
candidates = [File.join(Dir.pwd, CWD_CONFIG_NAME)]
|
|
81
|
+
begin
|
|
82
|
+
home = Dir.home
|
|
83
|
+
candidates << File.join(home, HOME_CONFIG_NAME) unless home.empty?
|
|
84
|
+
rescue ArgumentError
|
|
85
|
+
# No resolvable home directory; only the cwd candidate applies.
|
|
86
|
+
end
|
|
87
|
+
candidates.find { |path| File.file?(path) }
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# An absolute path with the home directory abbreviated to ~, for recording
|
|
91
|
+
# where an input came from without putting the username in shared reports.
|
|
92
|
+
module_function def display_source_path(path)
|
|
93
|
+
path = File.expand_path(path)
|
|
94
|
+
home = begin
|
|
95
|
+
File.expand_path(Dir.home)
|
|
96
|
+
rescue ArgumentError
|
|
97
|
+
nil
|
|
98
|
+
end
|
|
99
|
+
return path if home.nil? || home.empty? || home == File::SEPARATOR
|
|
100
|
+
return '~' if path == home
|
|
101
|
+
|
|
102
|
+
path.start_with?(home + File::SEPARATOR) ? "~#{path.delete_prefix(home)}" : path
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Make an identifier safe for a filename component.
|
|
106
|
+
module_function def sanitize(part)
|
|
107
|
+
part.to_s.gsub(/[^A-Za-z0-9._-]+/, '-').gsub(/\A[-.]+|[-.]+\z/, '')
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Meaningful query slug, derived from the prompt file name. The extension
|
|
111
|
+
# and then a trailing -prompt or _prompt (in any case) are stripped.
|
|
112
|
+
module_function def query_slug(prompt_path)
|
|
113
|
+
name = File.basename(prompt_path, File.extname(prompt_path)).sub(/[-_]prompt\z/i, '')
|
|
114
|
+
slug = sanitize(name).downcase
|
|
115
|
+
slug.empty? ? 'prompt' : slug
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# <run_stamp>_<query-slug>[_<index>]_<harness>_<provider>_<model>.md — the
|
|
119
|
+
# full timestamp and slug live in the filename so reports stay
|
|
120
|
+
# self-describing when copied out of the reports tree. index (an agent's
|
|
121
|
+
# 1-based position in the config's agent list) disambiguates repeated
|
|
122
|
+
# provider/model entries — e.g. the same model configured twice for two
|
|
123
|
+
# independent analyses — without depending on it being unique. It sits before the harness so that once more than one harness
|
|
124
|
+
# exists, filenames still sort in config/agent order first, rather than
|
|
125
|
+
# grouping by harness.
|
|
126
|
+
module_function def report_filename(file_prefix, provider, model, index: nil)
|
|
127
|
+
parts = [file_prefix]
|
|
128
|
+
parts << format('%02d', index) if index
|
|
129
|
+
parts << HARNESS::NAME
|
|
130
|
+
parts += [sanitize(provider), sanitize(model)]
|
|
131
|
+
"#{parts.join('_')}.md"
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Returns [label, ok, detail, usage]; never raises. usage is the run's
|
|
135
|
+
# token usage (see HARNESS), or nil; failed runs report it too, since
|
|
136
|
+
# their tokens were still spent.
|
|
137
|
+
# index: the agent's 1-based position in the config's agent list, used to
|
|
138
|
+
# disambiguate repeated provider/model entries in the filename and label;
|
|
139
|
+
# filename: overrides the output name; meta: becomes YAML front matter,
|
|
140
|
+
# which identifies the report; what was measured about its run (duration
|
|
141
|
+
# and usage) goes in run.json instead (see write_run_file);
|
|
142
|
+
# header: markdown prepended to the report body (or a callable receiving
|
|
143
|
+
# the run's fields, 'duration' (HH:MM:SS) and 'usage', and returning
|
|
144
|
+
# it). If the response begins
|
|
145
|
+
# with an H1 of its own, it is dropped: the header leads the body (and
|
|
146
|
+
# carries the title, in consolidated reports).
|
|
147
|
+
# pool: the WorkerPool to spawn the agent through, so Ctrl-C can stop it;
|
|
148
|
+
# without one it's spawned untracked.
|
|
149
|
+
module_function def run_agent(agent, prompt, run_dir, file_prefix, index: nil, filename: nil, meta: {},
|
|
150
|
+
header: nil, pool: nil)
|
|
151
|
+
provider = agent['provider'] if agent.is_a?(Hash)
|
|
152
|
+
model = agent['model'] if agent.is_a?(Hash)
|
|
153
|
+
label = agent_label(provider, model, index)
|
|
154
|
+
return [label, false, "config entry needs 'provider' and 'model'"] if provider.nil? || model.nil?
|
|
155
|
+
|
|
156
|
+
# Removed in the ensure below rather than via Dir.mktmpdir's block form,
|
|
157
|
+
# with errors ignored: a child that is still exiting (e.g. one just
|
|
158
|
+
# killed after the Ctrl-C grace period) can still be writing here
|
|
159
|
+
# (node's compile cache, say). That must not surface as an error.
|
|
160
|
+
scratch_dir = Dir.mktmpdir('agent-council-')
|
|
161
|
+
argv = HARNESS.argv(provider, model, scratch_dir)
|
|
162
|
+
(stdout, stderr, status), elapsed = timed do
|
|
163
|
+
(pool || Open3).capture3({ 'TMPDIR' => scratch_dir }, *argv, stdin_data: prompt)
|
|
164
|
+
end
|
|
165
|
+
# Harness output is UTF-8. WorkerPool#capture3 already returns it as such,
|
|
166
|
+
# but Open3.capture3 (used without a pool) returns the locale's encoding.
|
|
167
|
+
stdout, stderr = [stdout, stderr].map { |output| String.new(output, encoding: ENCODING) }
|
|
168
|
+
duration = format_duration(elapsed)
|
|
169
|
+
usage = HARNESS.usage(stdout)
|
|
170
|
+
|
|
171
|
+
unless status.success?
|
|
172
|
+
detail = stderr.lines.map(&:strip).reject(&:empty?).first || 'no error output'
|
|
173
|
+
return [label, false, "#{HARNESS::NAME} exited #{status.exitstatus}: #{detail}", usage]
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
error = HARNESS.error_message(stdout)
|
|
177
|
+
if error
|
|
178
|
+
summary = error.gsub(/\s+/, ' ').slice(0, 300)
|
|
179
|
+
return [label, false, "#{HARNESS::NAME} exited 0 but the run failed: #{summary}", usage]
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
text = HARNESS.final_response(stdout)
|
|
183
|
+
if text.nil? || text.strip.empty?
|
|
184
|
+
detail =
|
|
185
|
+
if HARNESS.assistant_messages(stdout).any?
|
|
186
|
+
'final assistant message had no text content (model may have answered in reasoning only)'
|
|
187
|
+
else
|
|
188
|
+
"no assistant message in #{HARNESS::NAME} event stream"
|
|
189
|
+
end
|
|
190
|
+
return [label, false, "#{HARNESS::NAME} exited 0 but #{detail}", usage]
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
filename ||= report_filename(file_prefix, provider, model, index: index)
|
|
194
|
+
run_fields = { 'duration' => duration, 'usage' => usage }.compact
|
|
195
|
+
if header
|
|
196
|
+
header = header.call(run_fields) if header.respond_to?(:call)
|
|
197
|
+
text = text.sub(/\A\s*#(?!#).+\n+/, '')
|
|
198
|
+
text = "#{header}\n#{text}"
|
|
199
|
+
end
|
|
200
|
+
content = with_front_matter(meta.merge('harness' => HARNESS::NAME,
|
|
201
|
+
'provider' => provider,
|
|
202
|
+
'model' => model), text)
|
|
203
|
+
# Exclusive create ('x'): never replaces a report, e.g. one written by
|
|
204
|
+
# a concurrent --consolidate with the same model; that is a failure.
|
|
205
|
+
File.write(File.join(run_dir, filename), content, mode: 'wx')
|
|
206
|
+
[label, true, filename, usage]
|
|
207
|
+
rescue => e
|
|
208
|
+
[label, false, "#{e.class}: #{e.message}", usage]
|
|
209
|
+
ensure
|
|
210
|
+
FileUtils.rm_rf(scratch_dir) if scratch_dir
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Creates a new run dir in day_dir and returns its name: base, or base-2,
|
|
214
|
+
# base-3, ... if that is taken. Dir.mkdir fails if the dir exists, so the
|
|
215
|
+
# claim is atomic: runs started in the same second (or whose local time
|
|
216
|
+
# repeats, e.g. when clocks fall back) never share a dir.
|
|
217
|
+
module_function def claim_run_dir(day_dir, base)
|
|
218
|
+
FileUtils.mkdir_p(day_dir)
|
|
219
|
+
(1..).each do |n|
|
|
220
|
+
name = n == 1 ? base : "#{base}-#{n}"
|
|
221
|
+
Dir.mkdir(File.join(day_dir, name))
|
|
222
|
+
return name
|
|
223
|
+
rescue Errno::EEXIST
|
|
224
|
+
next
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# The run dir's subdir holding the analysts' reports.
|
|
229
|
+
REVIEWS_DIR_NAME = 'reviews'
|
|
230
|
+
|
|
231
|
+
# A run's analyst reports as [source_label, body, fields] triples, sorted
|
|
232
|
+
# by filename; fields is the report's front matter. The dir is the run's
|
|
233
|
+
# reviews/ subdir; provider/model labels come from the YAML front matter
|
|
234
|
+
# run_agent writes — not from filename parsing. Files without parseable
|
|
235
|
+
# front matter naming a provider and model are not ours and are skipped.
|
|
236
|
+
module_function def collect_reports(reviews_dir)
|
|
237
|
+
# base: keeps reviews_dir from being read as a pattern, so a path with
|
|
238
|
+
# glob characters (e.g. "[") still finds its reports.
|
|
239
|
+
Dir.glob('*.md', base: reviews_dir).filter_map do |name|
|
|
240
|
+
fields, body = parse_front_matter(File.read(File.join(reviews_dir, name), encoding: ENCODING))
|
|
241
|
+
next unless fields && fields['provider'] && fields['model']
|
|
242
|
+
|
|
243
|
+
[agent_label(fields['provider'], fields['model'], fields['agent_index']), body, fields]
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# --- report front matter ---
|
|
248
|
+
|
|
249
|
+
module_function def with_front_matter(fields, body)
|
|
250
|
+
header = YAML.dump(fields.transform_keys(&:to_s)).delete_prefix("---\n").chomp
|
|
251
|
+
# Documented format: ISO 8601 timestamps, with or without a UTC offset,
|
|
252
|
+
# are unquoted (parse as Time).
|
|
253
|
+
iso = /\d{4}-\d\d-\d\dT[\d:]+(?:Z|[+-]\d\d:\d\d)?/
|
|
254
|
+
header = header.gsub(/^(timestamp|consolidated_at): ['"](#{iso})['"]$/, '\\1: \\2')
|
|
255
|
+
<<~TEXT
|
|
256
|
+
---
|
|
257
|
+
#{header}
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
#{body}
|
|
261
|
+
TEXT
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# [fields, body] from a report file, or [nil, nil] if it has no YAML
|
|
265
|
+
# front matter block (or the block cannot be safe-loaded).
|
|
266
|
+
module_function def parse_front_matter(text)
|
|
267
|
+
match = text.match(/\A---\n(.+?)\n---\n\n?/m)
|
|
268
|
+
return [nil, nil] unless match
|
|
269
|
+
|
|
270
|
+
fields =
|
|
271
|
+
begin
|
|
272
|
+
YAML.safe_load(match[1], permitted_classes: [Time])
|
|
273
|
+
rescue Psych::Exception
|
|
274
|
+
nil
|
|
275
|
+
end
|
|
276
|
+
return [nil, nil] unless fields.is_a?(Hash)
|
|
277
|
+
|
|
278
|
+
[fields, match.post_match]
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# The exact bytes piped to each analyst harness: the user's prompt, less
|
|
282
|
+
# one trailing newline, then a blank line and the report-format addendum.
|
|
283
|
+
module_function def compose_analyst_prompt(prompt, addendum)
|
|
284
|
+
"#{prompt.chomp}\n\n#{addendum}"
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# An agent's label in progress lines, report sources, and consolidation
|
|
288
|
+
# prompts: "#<index> <provider> / <model>", or without the index when
|
|
289
|
+
# there is none (the consolidator). missing_agents matches reports by it.
|
|
290
|
+
module_function def agent_label(provider, model, index = nil)
|
|
291
|
+
plain = "#{provider} / #{model}"
|
|
292
|
+
index ? "##{index} #{plain}" : plain
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# The consolidator's substantive prompt: the instructions template, the
|
|
296
|
+
# original prompt every analyst answered, all reports with their source
|
|
297
|
+
# labels, and the agents without a report, as [label, reason] pairs; the
|
|
298
|
+
# consolidator can't otherwise know about those.
|
|
299
|
+
module_function def consolidation_prompt(original_prompt, reports, failures = [])
|
|
300
|
+
template = File.read(CONSOLIDATE_PROMPT_PATH, encoding: ENCODING)
|
|
301
|
+
body = reports.map do |label, text, fields|
|
|
302
|
+
harness = (fields && fields['harness']) || HARNESS::NAME
|
|
303
|
+
"## Report from #{label}\n\nHarness: #{harness}\n\n#{text}"
|
|
304
|
+
end.join("\n\n---\n\n")
|
|
305
|
+
failed = failures.empty? ? 'None.' : failures.map { |label, reason| "- #{label}: #{reason}" }.join("\n")
|
|
306
|
+
<<~MARKDOWN
|
|
307
|
+
#{template}
|
|
308
|
+
|
|
309
|
+
---
|
|
310
|
+
|
|
311
|
+
# Original prompt (given identically to every analyst)
|
|
312
|
+
|
|
313
|
+
#{original_prompt}
|
|
314
|
+
|
|
315
|
+
---
|
|
316
|
+
|
|
317
|
+
# Analyst reports
|
|
318
|
+
|
|
319
|
+
#{body}
|
|
320
|
+
|
|
321
|
+
---
|
|
322
|
+
|
|
323
|
+
# Agents without a report
|
|
324
|
+
|
|
325
|
+
#{failed}
|
|
326
|
+
MARKDOWN
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# The consolidator's output filename; lives at the top level of the run
|
|
330
|
+
# dir, and stays self-describing when copied out of the tree.
|
|
331
|
+
# part: 'findings' or 'meta' for the files a consolidated report is split
|
|
332
|
+
# into (see split_consolidated_report); none for the report as written.
|
|
333
|
+
module_function def consolidated_filename(file_prefix, provider, model, part = nil)
|
|
334
|
+
suffix = part ? "_#{part}" : ''
|
|
335
|
+
"#{file_prefix}_consolidated_#{HARNESS::NAME}_#{sanitize(provider)}_#{sanitize(model)}#{suffix}.md"
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
# The line the consolidator is asked to write between its findings and
|
|
339
|
+
# the sections describing the consultation (see consolidate_prompt.md).
|
|
340
|
+
CONSOLIDATED_META_MARKER = '=== META ==='
|
|
341
|
+
|
|
342
|
+
# An analyst's label with its harness: "#<index> <harness> / <provider> /
|
|
343
|
+
# <model>", for the findings file's legend.
|
|
344
|
+
module_function def full_label(fields)
|
|
345
|
+
index = fields['agent_index']
|
|
346
|
+
parts = [fields['harness'] || HARNESS::NAME, fields['provider'], fields['model']].join(' / ')
|
|
347
|
+
index ? "##{index} #{parts}" : parts
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# [[findings, meta], nil] if text can be split on its marker line, else
|
|
351
|
+
# [nil, why]: a split that can't be trusted isn't made. The marker must occur once, with text on both sides and the
|
|
352
|
+
# Agent Consultants heading opening the second (the template puts that
|
|
353
|
+
# section first), and not inside a code fence, so that a marker the
|
|
354
|
+
# findings quote is not taken for the real one.
|
|
355
|
+
module_function def consolidated_split(text)
|
|
356
|
+
parts = text.split(/^#{Regexp.escape(CONSOLIDATED_META_MARKER)}[ \t\r]*$\n?/, -1)
|
|
357
|
+
line = "#{CONSOLIDATED_META_MARKER} line"
|
|
358
|
+
return [nil, "it has no #{line}"] if parts.size == 1
|
|
359
|
+
return [nil, "it has more than one #{line}"] if parts.size > 2
|
|
360
|
+
return [nil, "there is no text on one side of the #{line}"] if parts.any? { |part| part.strip.empty? }
|
|
361
|
+
unless parts.last.match?(/\A\s*## Agent Consultants/i)
|
|
362
|
+
return [nil, "the #{line} is not followed by an Agent Consultants section"]
|
|
363
|
+
end
|
|
364
|
+
return [nil, "the #{line} is inside a code fence"] if inside_code_fence?(parts.first)
|
|
365
|
+
|
|
366
|
+
[parts, nil]
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# True if text ends inside a fenced code block, going by CommonMark's
|
|
370
|
+
# rules for top-level fences: three or more backticks or tildes, closed by
|
|
371
|
+
# a fence of the same character at least as long. A backtick fence's info
|
|
372
|
+
# string can't contain a backtick. Fences nested in block quotes or deeply
|
|
373
|
+
# indented list items aren't tracked.
|
|
374
|
+
module_function def inside_code_fence?(text)
|
|
375
|
+
open_fence = nil
|
|
376
|
+
text.each_line do |line|
|
|
377
|
+
fence = line[/\A {0,3}(`{3,}|~{3,})/, 1] or next
|
|
378
|
+
if open_fence.nil?
|
|
379
|
+
open_fence = fence unless fence.start_with?('`') && line.delete_prefix(fence).include?('`')
|
|
380
|
+
elsif fence[0] == open_fence[0] && fence.size >= open_fence.size && line.strip == fence
|
|
381
|
+
open_fence = nil
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
!open_fence.nil?
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
# Splits the consolidated report written to filename (the meta file's
|
|
388
|
+
# name: run_agent creates it exclusively, so the name stays taken from
|
|
389
|
+
# then on and concurrent consolidations of the same configuration can't
|
|
390
|
+
# both succeed) in run_dir into a findings file, free of front matter and
|
|
391
|
+
# tables so it can be pasted into a chat as is, and the meta file with the
|
|
392
|
+
# rest, rewritten in place. head: the header run_agent put before the
|
|
393
|
+
# model's text, which stays in the meta file; labels: the analysts' full
|
|
394
|
+
# labels, for the findings file's legend; info: lines (without bullets)
|
|
395
|
+
# identifying the run, for its top. Returns the findings filename and the
|
|
396
|
+
# meta filename, or, if the report can't be split or the split files
|
|
397
|
+
# can't be written, the report under the unsplit name (see
|
|
398
|
+
# consolidated_filename) and nil, with a warning.
|
|
399
|
+
module_function def split_consolidated_report(run_dir, filename, file_prefix, provider, model, head,
|
|
400
|
+
labels, info = [])
|
|
401
|
+
path = File.join(run_dir, filename)
|
|
402
|
+
unsplit_name = consolidated_filename(file_prefix, provider, model)
|
|
403
|
+
findings_name = consolidated_filename(file_prefix, provider, model, 'findings')
|
|
404
|
+
written = []
|
|
405
|
+
content = File.read(path, encoding: ENCODING)
|
|
406
|
+
front = content[/\A---\n.+?\n---\n\n?/m] || ''
|
|
407
|
+
body = content.delete_prefix(front)
|
|
408
|
+
# The header run_agent put before the model's text must still be there,
|
|
409
|
+
# or it would end up in the findings file.
|
|
410
|
+
if head && !body.start_with?("#{head}\n")
|
|
411
|
+
warn "Could not split #{relative_path(path)}: its header is not what was written"
|
|
412
|
+
return keep_unsplit(run_dir, filename, unsplit_name)
|
|
413
|
+
end
|
|
414
|
+
parts, problem = consolidated_split(head ? body.delete_prefix("#{head}\n") : body)
|
|
415
|
+
unless parts
|
|
416
|
+
warn "Not splitting #{relative_path(path)}: #{problem}"
|
|
417
|
+
return keep_unsplit(run_dir, filename, unsplit_name)
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
findings, meta = parts
|
|
421
|
+
about = info.map { |line| "- #{line}\n" }.join
|
|
422
|
+
legend = labels.map { |label| "- #{label}" }.join("\n")
|
|
423
|
+
# Recorded once the file exists, even if writing it then fails, so
|
|
424
|
+
# cleanup removes a partial file; one that was already there (the
|
|
425
|
+
# exclusive create raises) is not ours to remove.
|
|
426
|
+
File.open(File.join(run_dir, findings_name), 'wx') do |file|
|
|
427
|
+
written << findings_name
|
|
428
|
+
file.write("# Consolidated Findings\n\n#{about}\nAgents consulted (agent number, harness, " \
|
|
429
|
+
"provider, model):\n\n#{legend}\n\n#{findings}")
|
|
430
|
+
end
|
|
431
|
+
# Replaced atomically, so the whole report survives a failed write.
|
|
432
|
+
temp = "#{path}.tmp"
|
|
433
|
+
written << File.basename(temp)
|
|
434
|
+
File.write(temp, "#{front}#{head}\n#{meta.lstrip}")
|
|
435
|
+
File.rename(temp, path)
|
|
436
|
+
[findings_name, filename]
|
|
437
|
+
rescue => e
|
|
438
|
+
# A failed split must not fail the consolidation, whose model run is
|
|
439
|
+
# already paid for and whose report is intact.
|
|
440
|
+
warn "Could not split #{relative_path(path)}: #{e.message}"
|
|
441
|
+
written.each { |name| FileUtils.rm_f(File.join(run_dir, name)) }
|
|
442
|
+
keep_unsplit(run_dir, filename, unsplit_name)
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
# Renames the report filename to unsplit_name, unless that is taken (then
|
|
446
|
+
# it stays as it is). Returns the name it ends up under and nil, as
|
|
447
|
+
# split_consolidated_report does for an unsplit report.
|
|
448
|
+
module_function def keep_unsplit(run_dir, filename, unsplit_name)
|
|
449
|
+
target = File.join(run_dir, unsplit_name)
|
|
450
|
+
if File.exist?(target)
|
|
451
|
+
warn "#{relative_path(target)} exists; leaving the report as #{filename}"
|
|
452
|
+
return [filename, nil]
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
File.rename(File.join(run_dir, filename), target)
|
|
456
|
+
[unsplit_name, nil]
|
|
457
|
+
rescue SystemCallError => e
|
|
458
|
+
warn "Could not rename #{filename} to #{unsplit_name} (#{e.message}); leaving the report as #{filename}"
|
|
459
|
+
[filename, nil]
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
# Markdown table of the report's main front-matter fields and its run's
|
|
463
|
+
# duration and usage, leading the body (under the title, in consolidated
|
|
464
|
+
# reports, followed by sources_table).
|
|
465
|
+
module_function def byline_table(meta, provider, model)
|
|
466
|
+
fields = {
|
|
467
|
+
'role' => meta['role'],
|
|
468
|
+
'agent' => (meta['agent_index'] ? "##{meta['agent_index']}" : nil),
|
|
469
|
+
'query' => meta['query'],
|
|
470
|
+
'harness' => HARNESS::NAME,
|
|
471
|
+
'provider' => provider,
|
|
472
|
+
'model' => model,
|
|
473
|
+
'timestamp' => meta['timestamp'],
|
|
474
|
+
'consolidated_at' => meta['consolidated_at'],
|
|
475
|
+
'duration' => meta['duration'],
|
|
476
|
+
'usage' => format_usage(meta['usage']),
|
|
477
|
+
}.compact
|
|
478
|
+
rows = fields.map { |field, value| "| #{field} | #{value} |" }
|
|
479
|
+
<<~TABLE
|
|
480
|
+
| Field | Value |
|
|
481
|
+
| :-------- | :---------------------------- |
|
|
482
|
+
#{rows.join("\n")}
|
|
483
|
+
TABLE
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
# Markdown table of a consolidated report's sources, one row per label in
|
|
487
|
+
# sources, with its run's duration and usage from source_runs (label =>
|
|
488
|
+
# run.json entry); - where unknown. Input counts cached input too.
|
|
489
|
+
module_function def sources_table(sources, source_runs)
|
|
490
|
+
rows = sources.map do |label|
|
|
491
|
+
run = source_runs[label] || {}
|
|
492
|
+
seconds = run['duration_seconds']
|
|
493
|
+
duration = seconds.is_a?(Numeric) ? format_duration(seconds) : '-'
|
|
494
|
+
"| #{[label, duration, *usage_cells(run['usage'])].join(' | ')} |"
|
|
495
|
+
end
|
|
496
|
+
<<~TABLE
|
|
497
|
+
### Sources
|
|
498
|
+
|
|
499
|
+
| Source | Duration | Input | Cached | Output | Cost |
|
|
500
|
+
| :----- | -------: | ----: | -----: | -----: | ---: |
|
|
501
|
+
#{rows.join("\n")}
|
|
502
|
+
TABLE
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
# Runs the configured consolidator model over a run's reports and writes
|
|
506
|
+
# its report into the same run dir. Returns as run_agent, except that
|
|
507
|
+
# detail is the findings file's name when the report was split (see
|
|
508
|
+
# split_consolidated_report) and the meta file's name follows usage;
|
|
509
|
+
# never raises.
|
|
510
|
+
# analyst_runs: the run's run.json analyst entries, for the sources
|
|
511
|
+
# table's durations and usage. failures: as for consolidation_prompt.
|
|
512
|
+
# pool: as for run_agent.
|
|
513
|
+
module_function def run_consolidation(consolidator, original_prompt, run_dir, file_prefix,
|
|
514
|
+
meta: {}, prompt_path: nil, failures: [], analyst_runs: [], pool: nil)
|
|
515
|
+
provider = consolidator['provider'] if consolidator.is_a?(Hash)
|
|
516
|
+
model = consolidator['model'] if consolidator.is_a?(Hash)
|
|
517
|
+
return ['consolidator', false, "config entry needs 'provider' and 'model'"] if provider.nil? || model.nil?
|
|
518
|
+
|
|
519
|
+
reports = collect_reports(File.join(run_dir, REVIEWS_DIR_NAME))
|
|
520
|
+
if reports.empty?
|
|
521
|
+
return [agent_label(provider, model), false,
|
|
522
|
+
"no reports found in #{relative_path(File.join(run_dir, REVIEWS_DIR_NAME))}"]
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
prompt = consolidation_prompt(original_prompt, reports, failures)
|
|
526
|
+
source_runs = analyst_runs.grep(Hash)
|
|
527
|
+
.to_h { |run| [agent_label(run['provider'], run['model'], run['agent_index']), run] }
|
|
528
|
+
# timestamp is the run's start; consolidated_at is this consolidation's,
|
|
529
|
+
# which is later when done by --consolidate.
|
|
530
|
+
meta = meta.merge('role' => 'consolidator', 'consolidated_at' => iso_timestamp(Time.now),
|
|
531
|
+
'sources' => reports.map(&:first))
|
|
532
|
+
prompt_line = prompt_path ? "### Prompt: #{prompt_path}\n\n" : ''
|
|
533
|
+
head = nil
|
|
534
|
+
header = ->(run_fields) do
|
|
535
|
+
head = "# Consolidated Report\n\n#{prompt_line}#{byline_table(meta.merge(run_fields), provider,
|
|
536
|
+
model)}\n" \
|
|
537
|
+
"#{sources_table(meta['sources'], source_runs)}"
|
|
538
|
+
end
|
|
539
|
+
label, ok, detail, usage = run_agent(consolidator, prompt, run_dir, file_prefix,
|
|
540
|
+
filename: consolidated_filename(file_prefix, provider, model, 'meta'),
|
|
541
|
+
header: header,
|
|
542
|
+
meta: meta,
|
|
543
|
+
pool: pool)
|
|
544
|
+
if ok
|
|
545
|
+
labels = reports.map { |_, _, fields| full_label(fields) }
|
|
546
|
+
info = ["Run: #{file_prefix}", prompt_path && "Prompt: #{File.basename(prompt_path)}",
|
|
547
|
+
"Consolidated by: #{HARNESS::NAME} / #{provider} / #{model}, #{meta['consolidated_at']}"].compact
|
|
548
|
+
detail, meta_name = split_consolidated_report(run_dir, detail, file_prefix, provider, model, head,
|
|
549
|
+
labels, info)
|
|
550
|
+
end
|
|
551
|
+
[label, ok, detail, usage, meta_name]
|
|
552
|
+
rescue => e
|
|
553
|
+
[agent_label(provider, model), false, "#{e.class}: #{e.message}"]
|
|
554
|
+
end
|
|
555
|
+
|
|
556
|
+
module_function def relative_path(path)
|
|
557
|
+
path = File.expand_path(path)
|
|
558
|
+
cwd = File.expand_path(Dir.pwd)
|
|
559
|
+
return path unless path == cwd || path.start_with?(cwd + File::SEPARATOR)
|
|
560
|
+
|
|
561
|
+
path.delete_prefix(cwd + File::SEPARATOR)
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
# The front matter's timestamp format: ISO 8601 with the UTC offset.
|
|
565
|
+
module_function def iso_timestamp(time)
|
|
566
|
+
time.strftime('%Y-%m-%dT%H:%M:%S%:z')
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
# Whole seconds as HH:MM:SS (hours may exceed 99); rounds to the nearest second.
|
|
570
|
+
module_function def format_duration(seconds)
|
|
571
|
+
total = seconds.round
|
|
572
|
+
format('%02d:%02d:%02d', total / 3600, total % 3600 / 60, total % 60)
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
# Token usage (see HARNESS) as e.g. "1.2M in (900.0k cached) / 30.5k out,
|
|
576
|
+
# $0.85", where "in" counts all input, cached or not. The cost is left out
|
|
577
|
+
# when unknown. nil for nil usage.
|
|
578
|
+
module_function def format_usage(usage)
|
|
579
|
+
input, cached, output, cost = usage_figures(usage)
|
|
580
|
+
return nil unless input
|
|
581
|
+
|
|
582
|
+
cached_note = " (#{format_token_count(cached)} cached)" if cached.positive?
|
|
583
|
+
text = "#{format_token_count(input)} in#{cached_note} / #{format_token_count(output)} out"
|
|
584
|
+
cost ? "#{text}, #{format_cost(cost)}" : text
|
|
585
|
+
end
|
|
586
|
+
|
|
587
|
+
# Display cells for a usage's input (cached included), cached input,
|
|
588
|
+
# output, and cost; - for each unknown one.
|
|
589
|
+
module_function def usage_cells(usage)
|
|
590
|
+
input, cached, output, cost = usage_figures(usage)
|
|
591
|
+
counts = [input, cached, output].map { |count| count ? format_token_count(count) : '-' }
|
|
592
|
+
[*counts, cost ? format_cost(cost) : '-']
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
# [input (cached or not), cached input, output, cost (nil when unknown)]
|
|
596
|
+
# from a usage hash (see HARNESS), or nil for unknown usage.
|
|
597
|
+
module_function def usage_figures(usage)
|
|
598
|
+
return nil unless usage.is_a?(Hash)
|
|
599
|
+
|
|
600
|
+
cached = usage['cache_read'].to_i
|
|
601
|
+
cost = usage['cost'] if usage['cost'].is_a?(Numeric) && usage['cost'].positive?
|
|
602
|
+
[usage['input'].to_i + cached + usage['cache_write'].to_i, cached, usage['output'].to_i, cost]
|
|
603
|
+
end
|
|
604
|
+
|
|
605
|
+
# 950, 45.2k, 1.2M.
|
|
606
|
+
module_function def format_token_count(count)
|
|
607
|
+
return count.to_s if count < 1000
|
|
608
|
+
|
|
609
|
+
count < 999_950 ? format('%.1fk', count / 1e3) : format('%.1fM', count / 1e6)
|
|
610
|
+
end
|
|
611
|
+
|
|
612
|
+
module_function def format_cost(dollars)
|
|
613
|
+
dollars < 0.01 ? '<$0.01' : format('$%.2f', dollars)
|
|
614
|
+
end
|
|
615
|
+
|
|
616
|
+
# The sum of usage hashes (see HARNESS), ignoring nils (and anything else
|
|
617
|
+
# not a hash); nil if there are none.
|
|
618
|
+
# The cost is summed only if every usage has one, so a partial total
|
|
619
|
+
# isn't presented as the whole.
|
|
620
|
+
module_function def sum_usage(usages)
|
|
621
|
+
usages = usages.grep(Hash)
|
|
622
|
+
return nil if usages.empty?
|
|
623
|
+
|
|
624
|
+
total = %w[input output cache_read cache_write].to_h { |key| [key, usages.sum { |u| u[key].to_i }] }
|
|
625
|
+
costs = usages.map { |u| u['cost'] }
|
|
626
|
+
costs.all?(Numeric) ? total.merge('cost' => costs.sum.round(6)) : total
|
|
627
|
+
end
|
|
628
|
+
|
|
629
|
+
# The run dir's machine-readable record of its agents' runs.
|
|
630
|
+
RUN_FILENAME = 'run.json'
|
|
631
|
+
|
|
632
|
+
# A run.json entry for an agent's run_agent result and its elapsed
|
|
633
|
+
# seconds: its report's path under dir (the run dir's reviews_dir for
|
|
634
|
+
# analysts) on success, its error on failure. The harness is per entry,
|
|
635
|
+
# not per run, so a run can mix harnesses. meta_report: for a
|
|
636
|
+
# consolidation whose report was split (its detail is the findings file's
|
|
637
|
+
# name), the other file, which carries the front matter.
|
|
638
|
+
module_function def run_entry(agent, ok, detail, elapsed, usage, index: nil, dir: nil, meta_report: nil)
|
|
639
|
+
entry = { 'agent_index' => index, 'harness' => HARNESS::NAME, 'provider' => agent['provider'],
|
|
640
|
+
'model' => agent['model'], 'ok' => ok }.compact
|
|
641
|
+
entry[ok ? 'report' : 'error'] = ok && dir ? File.join(dir, detail) : detail
|
|
642
|
+
entry['meta_report'] = (dir ? File.join(dir, meta_report) : meta_report) if ok && meta_report
|
|
643
|
+
entry.merge('duration_seconds' => elapsed.round(1), 'usage' => usage)
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
# Writes the run dir's run.json: every analyst's and consolidation's
|
|
647
|
+
# entry (see run_entry), their total usage, and how many runs it covers
|
|
648
|
+
# (those that reported usage). Replaced atomically, so a
|
|
649
|
+
# reader or a Ctrl-C never sees it half written. A failure to write it is
|
|
650
|
+
# only warned about: the reports are what matter.
|
|
651
|
+
module_function def write_run_file(run_dir, analysts, consolidations)
|
|
652
|
+
usages = (analysts + consolidations).map { |entry| entry['usage'] }
|
|
653
|
+
data = { 'analysts' => analysts, 'consolidations' => consolidations,
|
|
654
|
+
'total_usage' => sum_usage(usages), 'total_usage_runs' => usages.grep(Hash).size }
|
|
655
|
+
path = File.join(run_dir, RUN_FILENAME)
|
|
656
|
+
temp = "#{path}.#{Process.pid}.tmp"
|
|
657
|
+
File.write(temp, "#{JSON.pretty_generate(data)}\n")
|
|
658
|
+
File.rename(temp, path)
|
|
659
|
+
rescue SystemCallError => e
|
|
660
|
+
FileUtils.rm_f(temp) if temp
|
|
661
|
+
warn "Could not write #{relative_path(path)}: #{e.message}"
|
|
662
|
+
end
|
|
663
|
+
|
|
664
|
+
# [analysts, consolidations] from a run dir's run.json, or nil if it is
|
|
665
|
+
# missing or malformed.
|
|
666
|
+
module_function def read_run_file(run_dir)
|
|
667
|
+
data = JSON.parse(File.read(File.join(run_dir, RUN_FILENAME), encoding: ENCODING))
|
|
668
|
+
lists = data.values_at('analysts', 'consolidations') if data.is_a?(Hash)
|
|
669
|
+
lists if lists&.all?(Array)
|
|
670
|
+
rescue SystemCallError, JSON::ParserError
|
|
671
|
+
nil
|
|
672
|
+
end
|
|
673
|
+
|
|
674
|
+
# Runs the block; returns [block_result, elapsed_seconds] (monotonic clock).
|
|
675
|
+
module_function def timed
|
|
676
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
677
|
+
result = yield
|
|
678
|
+
[result, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started]
|
|
679
|
+
end
|
|
680
|
+
|
|
681
|
+
RESULT_HEADINGS = %w[Ok Duration Agent In Cached Out Cost].freeze
|
|
682
|
+
# The usage columns, right-aligned so their figures line up.
|
|
683
|
+
RIGHT_ALIGNED_COLUMNS = (3..6)
|
|
684
|
+
|
|
685
|
+
# Indented result table lines for [label, ok, detail, elapsed, usage]
|
|
686
|
+
# results: a heading row, then one row per result, with ✓ or ✗ and its
|
|
687
|
+
# usage (as for usage_cells), then the failures' errors, if any, listed
|
|
688
|
+
# below the table so long messages don't widen it. Report filenames
|
|
689
|
+
# aren't shown: an analyst's is found by its agent number, and the
|
|
690
|
+
# consolidated report is at the run dir's top level.
|
|
691
|
+
module_function def format_result_table(results)
|
|
692
|
+
rows = results.map do |label, ok, _detail, elapsed, usage|
|
|
693
|
+
[ok ? '✓' : '✗', format_duration(elapsed), label, *usage_cells(usage)]
|
|
694
|
+
end
|
|
695
|
+
table = [RESULT_HEADINGS, *rows]
|
|
696
|
+
widths = table.transpose.map { |column| column.map { |cell| display_width(cell) }.max }
|
|
697
|
+
lines = table.map do |row|
|
|
698
|
+
cells = row.each_with_index.map do |cell, i|
|
|
699
|
+
padding = ' ' * (widths[i] - display_width(cell))
|
|
700
|
+
RIGHT_ALIGNED_COLUMNS.cover?(i) ? padding + cell : cell + padding
|
|
701
|
+
end
|
|
702
|
+
" #{cells.join(' ')}".rstrip
|
|
703
|
+
end
|
|
704
|
+
errors = results.reject { |_, ok| ok }.map { |label, _, detail| " #{label}: #{detail}" }
|
|
705
|
+
errors.empty? ? lines : [*lines, '', ' Errors:', *errors]
|
|
706
|
+
end
|
|
707
|
+
|
|
708
|
+
# "; <total usage>" for a summary line, noting how many of the runs it
|
|
709
|
+
# covers when some reported no usage; "" when none reported any.
|
|
710
|
+
module_function def usage_summary(usages)
|
|
711
|
+
total = format_usage(sum_usage(usages))
|
|
712
|
+
return '' unless total
|
|
713
|
+
|
|
714
|
+
reported = usages.grep(Hash).size
|
|
715
|
+
"; #{total}#{" (from #{reported} of #{usages.size} agents)" if reported < usages.size}"
|
|
716
|
+
end
|
|
717
|
+
|
|
718
|
+
# East Asian wide and fullwidth characters (CJK, Hangul, fullwidth forms,
|
|
719
|
+
# most emoji), which terminals show two columns wide.
|
|
720
|
+
WIDE_CHAR = Regexp.new(
|
|
721
|
+
'[\u1100-\u115F\u2E80-\u303E\u3041-\u33FF\u3400-\u4DBF\u4E00-\u9FFF' \
|
|
722
|
+
'\uA000-\uA4CF\uAC00-\uD7A3\uF900-\uFAFF\uFE30-\uFE4F\uFF00-\uFF60' \
|
|
723
|
+
'\uFFE0-\uFFE6\u{1F300}-\u{1F64F}\u{1F900}-\u{1F9FF}\u{20000}-\u{3FFFD}]'
|
|
724
|
+
)
|
|
725
|
+
# Combining marks and zero-width characters, which take no column.
|
|
726
|
+
ZERO_WIDTH_CHAR = /[\p{Mn}\p{Me}\u200B-\u200F\uFEFF]/
|
|
727
|
+
|
|
728
|
+
# The number of terminal columns text takes: as String#length, but
|
|
729
|
+
# counting wide characters as two and zero-width ones as none.
|
|
730
|
+
# Approximate (e.g. ambiguous-width characters count as one), but enough
|
|
731
|
+
# to keep result tables aligned.
|
|
732
|
+
module_function def display_width(text)
|
|
733
|
+
text.length + text.scan(WIDE_CHAR).size - text.scan(ZERO_WIDTH_CHAR).size
|
|
734
|
+
end
|
|
735
|
+
|
|
736
|
+
# Prints the "--- Consolidation complete ---" (or error) marker.
|
|
737
|
+
module_function def puts_consolidation_status(ok)
|
|
738
|
+
puts "\n--- Consolidation #{ok ? 'complete' : 'error'} ---"
|
|
739
|
+
end
|
|
740
|
+
|
|
741
|
+
# True if a config entry (agent or consolidator) has the required string
|
|
742
|
+
# 'provider' and 'model' fields.
|
|
743
|
+
module_function def valid_agent_entry?(entry)
|
|
744
|
+
entry.is_a?(Hash) && entry['provider'].is_a?(String) && entry['model'].is_a?(String)
|
|
745
|
+
end
|
|
746
|
+
|
|
747
|
+
# True if `name` (a bare command name) is an executable file in a PATH
|
|
748
|
+
# directory. Checked up front so a missing harness fails before any run
|
|
749
|
+
# dir is created, rather than as an Errno::ENOENT from every agent. An
|
|
750
|
+
# empty PATH entry means the current directory, as it does for execvp (-1
|
|
751
|
+
# keeps a trailing empty entry).
|
|
752
|
+
module_function def executable_on_path?(name)
|
|
753
|
+
ENV.fetch('PATH', '').split(File::PATH_SEPARATOR, -1).map { |dir| dir.empty? ? '.' : dir }.any? do |dir|
|
|
754
|
+
path = File.join(dir, name)
|
|
755
|
+
File.executable?(path) && !File.directory?(path)
|
|
756
|
+
end
|
|
757
|
+
end
|
|
758
|
+
|
|
759
|
+
# "harness / provider / model" for a validated config entry, printed at
|
|
760
|
+
# startup so it is clear which agents are still running later.
|
|
761
|
+
module_function def agent_descriptor(entry)
|
|
762
|
+
"#{HARNESS::NAME} / #{entry['provider']} / #{entry['model']}"
|
|
763
|
+
end
|
|
764
|
+
|
|
765
|
+
# AGENT_COUNCIL_OPTS as UTF-8, or nil when unset. Retagged because ENV
|
|
766
|
+
# values carry the locale's encoding (binary under LANG=C), which would let
|
|
767
|
+
# invalid bytes through unchecked.
|
|
768
|
+
module_function def env_options_value(env = ENV)
|
|
769
|
+
env[ENV_OPTS_VAR]&.dup&.force_encoding(Encoding::UTF_8)
|
|
770
|
+
end
|
|
771
|
+
|
|
772
|
+
# Shell-split tokens from AGENT_COUNCIL_OPTS, or [] when unset.
|
|
773
|
+
module_function def env_options(env = ENV)
|
|
774
|
+
raw = env_options_value(env)
|
|
775
|
+
return [] if raw.nil? || raw.strip.empty?
|
|
776
|
+
|
|
777
|
+
Shellwords.shellsplit(raw)
|
|
778
|
+
rescue ArgumentError, EncodingError => e # e.g. an unclosed quote, invalid UTF-8
|
|
779
|
+
raise ArgumentError, "Invalid #{ENV_OPTS_VAR} value: #{e.message}"
|
|
780
|
+
end
|
|
781
|
+
|
|
782
|
+
# options: receives the parsed option values (:consolidate, :output_dir).
|
|
783
|
+
# env: where -h finds the current AGENT_COUNCIL_OPTS value.
|
|
784
|
+
module_function def option_parser(options = {}, env = ENV)
|
|
785
|
+
OptionParser.new do |opts|
|
|
786
|
+
opts.banner = <<~BANNER
|
|
787
|
+
-------------------------------------------------------------------------------
|
|
788
|
+
Usage: agent-council [options] PROMPT_FILE [CONFIG_FILE]
|
|
789
|
+
agent-council --consolidate RUN_DIR
|
|
790
|
+
Repository: https://github.com/keithrbennett/agent-council
|
|
791
|
+
Documentation (Local): #{File.join(PROJECT_ROOT, 'README.md')}
|
|
792
|
+
Examples (Local): #{EXAMPLES_DIR}
|
|
793
|
+
Version: #{VERSION}
|
|
794
|
+
-------------------------------------------------------------------------------
|
|
795
|
+
|
|
796
|
+
Arguments:
|
|
797
|
+
PROMPT_FILE Path to the prompt file to send to every configured model (required)
|
|
798
|
+
CONFIG_FILE Path to a YAML agent config (default: ./#{CWD_CONFIG_NAME}, else ~/#{HOME_CONFIG_NAME}; required if neither exists)
|
|
799
|
+
RUN_DIR With --consolidate, a finished run's directory (no PROMPT_FILE or CONFIG_FILE)
|
|
800
|
+
|
|
801
|
+
Agents analyze the current directory: run from the project to be analyzed.
|
|
802
|
+
|
|
803
|
+
Options:
|
|
804
|
+
BANNER
|
|
805
|
+
|
|
806
|
+
opts.on('--consolidate RUN_DIR',
|
|
807
|
+
'Consolidate an existing run\'s reports, in place; runs no analysts') do |dir|
|
|
808
|
+
options[:consolidate] = dir
|
|
809
|
+
end
|
|
810
|
+
|
|
811
|
+
opts.on('-o', '--output-dir DIR',
|
|
812
|
+
'Base directory for run dirs, as DIR/<date>/<time>',
|
|
813
|
+
"(default: ./#{DEFAULT_REPORTS_DIR_NAME})",
|
|
814
|
+
'Not allowed with --consolidate') do |dir|
|
|
815
|
+
options[:output_dir] = dir
|
|
816
|
+
end
|
|
817
|
+
|
|
818
|
+
opts.on('-h', '--help', 'Show this help and exit') do
|
|
819
|
+
puts opts
|
|
820
|
+
exit 0
|
|
821
|
+
end
|
|
822
|
+
|
|
823
|
+
opts.on('-v', '--version', 'Show the version and exit') do
|
|
824
|
+
puts "agent-council #{VERSION}"
|
|
825
|
+
exit 0
|
|
826
|
+
end
|
|
827
|
+
|
|
828
|
+
opts.separator ''
|
|
829
|
+
opts.separator 'Environment:'
|
|
830
|
+
opts.separator " #{ENV_OPTS_VAR} holds arguments inserted before the command-line ones,"
|
|
831
|
+
opts.separator ' so an option given on the command line overrides the same option there.'
|
|
832
|
+
opts.separator " Example: export #{ENV_OPTS_VAR}=\"--output-dir tmp/council\""
|
|
833
|
+
# Inspected, so newlines can't break -h. (A value with invalid bytes
|
|
834
|
+
# never gets here: env_options rejects it first.)
|
|
835
|
+
env_value = env_options_value(env).to_s
|
|
836
|
+
opts.separator " Currently set to: #{env_value.inspect}" unless env_value.strip.empty?
|
|
837
|
+
opts.separator ''
|
|
838
|
+
opts.separator 'Examples:'
|
|
839
|
+
opts.separator ' agent-council my-analysis-prompt.md'
|
|
840
|
+
opts.separator ' agent-council my-analysis-prompt.md other-config.yml'
|
|
841
|
+
opts.separator ' agent-council -o tmp/council my-analysis-prompt.md'
|
|
842
|
+
opts.separator " agent-council --consolidate #{DEFAULT_REPORTS_DIR_NAME}/2026-09-03/09-14-02"
|
|
843
|
+
end
|
|
844
|
+
end
|
|
845
|
+
|
|
846
|
+
module_function def main(argv)
|
|
847
|
+
abort_if_native_windows
|
|
848
|
+
options = {}
|
|
849
|
+
parser = option_parser(options)
|
|
850
|
+
begin
|
|
851
|
+
argv = parser.parse(env_options + argv)
|
|
852
|
+
rescue ArgumentError => e # a malformed AGENT_COUNCIL_OPTS
|
|
853
|
+
abort e.message
|
|
854
|
+
rescue OptionParser::ParseError => e
|
|
855
|
+
warn e.message
|
|
856
|
+
warn parser
|
|
857
|
+
exit 1
|
|
858
|
+
end
|
|
859
|
+
|
|
860
|
+
if options[:consolidate]
|
|
861
|
+
unless argv.empty?
|
|
862
|
+
warn parser
|
|
863
|
+
exit 1
|
|
864
|
+
end
|
|
865
|
+
# The consolidation report goes into RUN_DIR itself.
|
|
866
|
+
if options.key?(:output_dir)
|
|
867
|
+
abort "--output-dir (given directly or in #{ENV_OPTS_VAR}) cannot be used with --consolidate, " \
|
|
868
|
+
'which writes into RUN_DIR'
|
|
869
|
+
end
|
|
870
|
+
consolidate_run(options[:consolidate])
|
|
871
|
+
end
|
|
872
|
+
|
|
873
|
+
# Checked up front, so a bad value fails before any work, with a message
|
|
874
|
+
# rather than a backtrace. A blank value (e.g. -o "$UNSET_VAR") would
|
|
875
|
+
# otherwise resolve to the cwd and scatter day dirs over the project.
|
|
876
|
+
if options[:output_dir]&.strip&.empty?
|
|
877
|
+
abort '--output-dir needs a directory; got an empty value'
|
|
878
|
+
end
|
|
879
|
+
reports_root =
|
|
880
|
+
begin
|
|
881
|
+
reports_dir(options[:output_dir])
|
|
882
|
+
rescue ArgumentError => e # e.g. ~nosuchuser/x
|
|
883
|
+
abort "Invalid --output-dir #{options[:output_dir]}: #{e.message}"
|
|
884
|
+
end
|
|
885
|
+
# mkdir_p would fail on a file anywhere along the path, so check the
|
|
886
|
+
# nearest part that exists, which is the dir itself if it exists.
|
|
887
|
+
existing = reports_root
|
|
888
|
+
existing = File.dirname(existing) until File.exist?(existing)
|
|
889
|
+
unless File.directory?(existing)
|
|
890
|
+
abort "Output dir #{relative_path(reports_root)} can't be created: " \
|
|
891
|
+
"#{relative_path(existing)} is not a directory"
|
|
892
|
+
end
|
|
893
|
+
|
|
894
|
+
prompt_path = argv[0]
|
|
895
|
+
|
|
896
|
+
if prompt_path.nil? || argv.length > 2
|
|
897
|
+
warn parser
|
|
898
|
+
exit 1
|
|
899
|
+
end
|
|
900
|
+
abort "Prompt file not found: #{prompt_path}" unless File.file?(prompt_path)
|
|
901
|
+
|
|
902
|
+
config_path = resolve_config_path(argv[1])
|
|
903
|
+
unless config_path
|
|
904
|
+
abort <<~MESSAGE
|
|
905
|
+
No config file given, and neither ./#{CWD_CONFIG_NAME} nor ~/#{HOME_CONFIG_NAME} exists.
|
|
906
|
+
Pass a config path: agent-council PROMPT_FILE CONFIG_FILE
|
|
907
|
+
Minimal config: agents: [{provider: ..., model: ...}]
|
|
908
|
+
Sample configs: #{File.join(EXAMPLES_DIR, 'configs')}
|
|
909
|
+
MESSAGE
|
|
910
|
+
end
|
|
911
|
+
abort "Config file not found: #{config_path}" unless File.file?(config_path)
|
|
912
|
+
|
|
913
|
+
# Read once. Analysts receive the user's prompt followed by the
|
|
914
|
+
# report-format addendum (see compose_analyst_prompt); the consolidator
|
|
915
|
+
# cites the prompt alone.
|
|
916
|
+
# The prompt and analyst_format.md are copied into the run dir, so the
|
|
917
|
+
# exact bytes sent to each analyst are recoverable.
|
|
918
|
+
prompt = File.read(prompt_path, encoding: ENCODING)
|
|
919
|
+
abort "Prompt file is empty: #{prompt_path}" if prompt.empty?
|
|
920
|
+
abort "Analyst format addendum missing: #{ANALYST_FORMAT_PATH}" unless File.file?(ANALYST_FORMAT_PATH)
|
|
921
|
+
# All three are copied into the run dir under their own names, so a
|
|
922
|
+
# shared name would leave one copy in place of another.
|
|
923
|
+
copied_paths = { 'prompt' => prompt_path, 'config' => config_path, 'addendum' => ANALYST_FORMAT_PATH }
|
|
924
|
+
copied_names = copied_paths.transform_values { |path| File.basename(path) }
|
|
925
|
+
unless copied_names.values.uniq.size == copied_names.size
|
|
926
|
+
abort "The prompt, config, and #{File.basename(ANALYST_FORMAT_PATH)} need different file names, " \
|
|
927
|
+
'since all three are copied into the run dir: ' \
|
|
928
|
+
"#{copied_names.map { |role, name| "#{role} #{name}" }.join(', ')}"
|
|
929
|
+
end
|
|
930
|
+
if (role = copied_names.key(REVIEWS_DIR_NAME))
|
|
931
|
+
abort "The #{role} can't be named #{REVIEWS_DIR_NAME}, the run dir's subdirectory for analyst reports"
|
|
932
|
+
end
|
|
933
|
+
analyst_prompt = compose_analyst_prompt(prompt, File.read(ANALYST_FORMAT_PATH, encoding: ENCODING))
|
|
934
|
+
|
|
935
|
+
config =
|
|
936
|
+
begin
|
|
937
|
+
YAML.safe_load_file(config_path)
|
|
938
|
+
rescue Psych::Exception => e
|
|
939
|
+
abort "Invalid YAML in #{config_path}: #{e.message}"
|
|
940
|
+
end
|
|
941
|
+
abort "Config file must be a YAML mapping: #{config_path}" unless config.is_a?(Hash)
|
|
942
|
+
|
|
943
|
+
agents = config['agents']
|
|
944
|
+
abort "No agents defined in #{config_path}" unless agents.is_a?(Array) && !agents.empty?
|
|
945
|
+
unless agents.all? { |a| valid_agent_entry?(a) }
|
|
946
|
+
abort "Each agent in #{config_path} needs string 'provider' and 'model' fields"
|
|
947
|
+
end
|
|
948
|
+
|
|
949
|
+
consolidator = config['consolidator']
|
|
950
|
+
if consolidator
|
|
951
|
+
unless valid_agent_entry?(consolidator)
|
|
952
|
+
abort "Consolidator in #{config_path} needs string 'provider' and 'model' fields"
|
|
953
|
+
end
|
|
954
|
+
unless File.file?(CONSOLIDATE_PROMPT_PATH)
|
|
955
|
+
abort "Consolidator configured but template missing: #{CONSOLIDATE_PROMPT_PATH}"
|
|
956
|
+
end
|
|
957
|
+
end
|
|
958
|
+
|
|
959
|
+
abort_unless_harness_installed
|
|
960
|
+
|
|
961
|
+
# One run dir per invocation, day-bucketed so the reports dir stays
|
|
962
|
+
# navigable at volume: <reports dir>/<yyyy-mm-dd>/<hh-mm-ss>/. The dir name is pure time
|
|
963
|
+
# (plus a collision suffix, see claim_run_dir); full timestamp and query
|
|
964
|
+
# slug go into the report filenames (file_prefix).
|
|
965
|
+
now = Time.now
|
|
966
|
+
day = now.strftime('%Y-%m-%d')
|
|
967
|
+
day_dir = File.join(reports_root, day)
|
|
968
|
+
run_name =
|
|
969
|
+
begin
|
|
970
|
+
claim_run_dir(day_dir, now.strftime('%H-%M-%S'))
|
|
971
|
+
rescue SystemCallError => e # e.g. no write permission
|
|
972
|
+
abort "Cannot create a run dir under #{relative_path(reports_root)}: #{e.message}"
|
|
973
|
+
end
|
|
974
|
+
run_stamp = "#{day}-#{run_name}"
|
|
975
|
+
iso_stamp = iso_timestamp(now)
|
|
976
|
+
slug = query_slug(prompt_path)
|
|
977
|
+
file_prefix = "#{run_stamp}_#{slug}"
|
|
978
|
+
# prompt_file and config_file name the input copies in the run dir, and
|
|
979
|
+
# run_stamp gives the report filenames' prefix, so --consolidate needn't
|
|
980
|
+
# rely on the run dir's path.
|
|
981
|
+
report_meta = { 'role' => 'analyst', 'timestamp' => iso_stamp, 'query' => slug,
|
|
982
|
+
'prompt_file' => File.basename(prompt_path), 'config_file' => File.basename(config_path),
|
|
983
|
+
'config_source' => display_source_path(config_path), 'run_stamp' => run_stamp }
|
|
984
|
+
run_dir = File.join(day_dir, run_name)
|
|
985
|
+
reviews_dir = File.join(run_dir, REVIEWS_DIR_NAME)
|
|
986
|
+
FileUtils.mkdir_p(reviews_dir)
|
|
987
|
+
|
|
988
|
+
# Keep the run's inputs recoverable: copy the prompt, format
|
|
989
|
+
# addendum, and config. (The orchestrator is not copied: it is
|
|
990
|
+
# versioned in git.)
|
|
991
|
+
FileUtils.cp(prompt_path, run_dir)
|
|
992
|
+
FileUtils.cp(ANALYST_FORMAT_PATH, run_dir)
|
|
993
|
+
FileUtils.cp(config_path, run_dir)
|
|
994
|
+
|
|
995
|
+
# A default config's path is absolute (resolve_config_path makes it
|
|
996
|
+
# so), to show which of the defaults was picked up.
|
|
997
|
+
puts
|
|
998
|
+
puts(argv[1] ? "Config: #{relative_path(config_path)}" : "Config: [default] #{config_path}")
|
|
999
|
+
puts "Prompt: #{relative_path(prompt_path)}"
|
|
1000
|
+
puts "Run Dir: #{relative_path(run_dir)}"
|
|
1001
|
+
puts "\nReview Agents (#{agents.size}):"
|
|
1002
|
+
agents.each_with_index do |agent, i|
|
|
1003
|
+
puts " ##{i + 1} #{agent_descriptor(agent)}"
|
|
1004
|
+
end
|
|
1005
|
+
puts "\nConsolidator Agent:\n #{agent_descriptor(consolidator)}" if consolidator
|
|
1006
|
+
puts "\nReview Progress:"
|
|
1007
|
+
pool = WorkerPool.new(grace: INTERRUPT_GRACE_SECONDS)
|
|
1008
|
+
pool.trap_signals
|
|
1009
|
+
agents.each_with_index do |agent, i|
|
|
1010
|
+
index = i + 1
|
|
1011
|
+
pool.start do
|
|
1012
|
+
provider = agent.is_a?(Hash) ? agent['provider'] : nil
|
|
1013
|
+
model = agent.is_a?(Hash) ? agent['model'] : nil
|
|
1014
|
+
meta = report_meta.merge('agent_index' => index)
|
|
1015
|
+
(label, ok, detail, usage), elapsed = timed do
|
|
1016
|
+
run_agent(agent, analyst_prompt, reviews_dir, file_prefix,
|
|
1017
|
+
index: index,
|
|
1018
|
+
meta: meta,
|
|
1019
|
+
header: ->(run_fields) { byline_table(meta.merge(run_fields), provider, model) },
|
|
1020
|
+
pool: pool)
|
|
1021
|
+
end
|
|
1022
|
+
[label, ok, detail, elapsed, usage]
|
|
1023
|
+
end
|
|
1024
|
+
end
|
|
1025
|
+
results = pool.collect do |label, ok, detail, elapsed|
|
|
1026
|
+
duration = format_duration(elapsed)
|
|
1027
|
+
puts(ok ? " ✓ #{duration} #{label}" : " ✗ #{duration} #{label}\n #{detail}")
|
|
1028
|
+
end
|
|
1029
|
+
|
|
1030
|
+
failed = results.count { |_, ok, _| !ok }
|
|
1031
|
+
puts "\nReviews (in #{relative_path(reviews_dir)}):"
|
|
1032
|
+
puts format_result_table(results)
|
|
1033
|
+
puts "\n--- Reviews complete (#{results.size - failed}/#{results.size} succeeded" \
|
|
1034
|
+
"#{usage_summary(results.map(&:last))}) ---"
|
|
1035
|
+
# Written now, so it's there even if the consolidation is interrupted.
|
|
1036
|
+
analyst_runs = agents.zip(results).each_with_index.map do |(agent, (_, ok, detail, elapsed, usage)), i|
|
|
1037
|
+
run_entry(agent, ok, detail, elapsed, usage, index: i + 1, dir: REVIEWS_DIR_NAME)
|
|
1038
|
+
end
|
|
1039
|
+
write_run_file(run_dir, analyst_runs, [])
|
|
1040
|
+
|
|
1041
|
+
if consolidator
|
|
1042
|
+
puts "\nConsolidation Report (in #{relative_path(run_dir)}):"
|
|
1043
|
+
failures = results.reject { |_, ok, _| ok }.map { |label, _, detail, _| [label, detail] }
|
|
1044
|
+
pool.start do
|
|
1045
|
+
timed do
|
|
1046
|
+
run_consolidation(consolidator, prompt, run_dir, file_prefix,
|
|
1047
|
+
meta: report_meta, prompt_path: prompt_path, failures: failures, analyst_runs: analyst_runs,
|
|
1048
|
+
pool: pool)
|
|
1049
|
+
end
|
|
1050
|
+
end
|
|
1051
|
+
(label, ok, detail, usage, meta_name), elapsed = pool.collect.first
|
|
1052
|
+
puts format_result_table([[label, ok, detail, elapsed, usage]])
|
|
1053
|
+
entry = run_entry(consolidator, ok, detail, elapsed, usage, meta_report: meta_name)
|
|
1054
|
+
write_run_file(run_dir, analyst_runs, [entry])
|
|
1055
|
+
puts_consolidation_status(ok)
|
|
1056
|
+
end
|
|
1057
|
+
|
|
1058
|
+
exit(failed.zero? ? 0 : 1)
|
|
1059
|
+
rescue Interrupt
|
|
1060
|
+
exit_interrupted(pool)
|
|
1061
|
+
end
|
|
1062
|
+
|
|
1063
|
+
# --consolidate RUN_DIR: runs the consolidator over an existing
|
|
1064
|
+
# run's reports, writing its report into that run dir. The prompt,
|
|
1065
|
+
# config, and run's identity (timestamp, query, run_stamp) all come from
|
|
1066
|
+
# the run itself: its analysts' front matter names the prompt and config
|
|
1067
|
+
# copies.
|
|
1068
|
+
# The consolidator is the config copy's; edit that copy to try another.
|
|
1069
|
+
# Exits 0 only if the consolidation succeeded.
|
|
1070
|
+
module_function def consolidate_run(run_dir)
|
|
1071
|
+
abort "Run dir not found: #{run_dir}" unless File.directory?(run_dir)
|
|
1072
|
+
reviews_dir = File.join(run_dir, REVIEWS_DIR_NAME)
|
|
1073
|
+
reports = File.directory?(reviews_dir) ? collect_reports(reviews_dir) : []
|
|
1074
|
+
abort "No analyst reports found in #{reviews_dir}" if reports.empty?
|
|
1075
|
+
|
|
1076
|
+
# From the first report that has them, so one stray or older report in
|
|
1077
|
+
# reviews/ doesn't block the run.
|
|
1078
|
+
fields = reports.map { |report| report[2] }
|
|
1079
|
+
.find { |f| f.values_at('prompt_file', 'config_file', 'run_stamp', 'query').all? }
|
|
1080
|
+
unless fields
|
|
1081
|
+
abort "Reports in #{reviews_dir} lack prompt_file, config_file, run_stamp, or query; " \
|
|
1082
|
+
'runs made before --consolidate existed cannot be consolidated'
|
|
1083
|
+
end
|
|
1084
|
+
%w[prompt_file config_file].each do |key|
|
|
1085
|
+
abort "Invalid #{key} in #{reviews_dir}: #{fields[key].inspect}" unless bare_name?(fields[key])
|
|
1086
|
+
end
|
|
1087
|
+
prompt_path = File.join(run_dir, fields['prompt_file'])
|
|
1088
|
+
config_path = File.join(run_dir, fields['config_file'])
|
|
1089
|
+
abort "Prompt copy not found: #{prompt_path}" unless File.file?(prompt_path)
|
|
1090
|
+
abort "Config copy not found: #{config_path}" unless File.file?(config_path)
|
|
1091
|
+
|
|
1092
|
+
config =
|
|
1093
|
+
begin
|
|
1094
|
+
YAML.safe_load_file(config_path)
|
|
1095
|
+
rescue Psych::Exception => e
|
|
1096
|
+
abort "Invalid YAML in #{config_path}: #{e.message}"
|
|
1097
|
+
end
|
|
1098
|
+
consolidator = config['consolidator'] if config.is_a?(Hash)
|
|
1099
|
+
abort "No consolidator in #{config_path}; add one to consolidate" unless consolidator
|
|
1100
|
+
unless valid_agent_entry?(consolidator)
|
|
1101
|
+
abort "Consolidator in #{config_path} needs string 'provider' and 'model' fields"
|
|
1102
|
+
end
|
|
1103
|
+
unless File.file?(CONSOLIDATE_PROMPT_PATH)
|
|
1104
|
+
abort "Consolidation template missing: #{CONSOLIDATE_PROMPT_PATH}"
|
|
1105
|
+
end
|
|
1106
|
+
|
|
1107
|
+
file_prefix = "#{fields['run_stamp']}_#{fields['query']}" # as in a normal run
|
|
1108
|
+
filename = consolidated_filename(file_prefix, consolidator['provider'], consolidator['model'])
|
|
1109
|
+
unless bare_name?(filename)
|
|
1110
|
+
abort "Invalid run_stamp or query in #{reviews_dir}: #{fields.values_at('run_stamp', 'query').inspect}"
|
|
1111
|
+
end
|
|
1112
|
+
names = [nil, 'findings', 'meta'].map do |part|
|
|
1113
|
+
consolidated_filename(file_prefix, consolidator['provider'], consolidator['model'], part)
|
|
1114
|
+
end
|
|
1115
|
+
existing = names.select { |name| File.exist?(File.join(run_dir, name)) }
|
|
1116
|
+
unless existing.empty?
|
|
1117
|
+
listed = existing.map { |name| relative_path(File.join(run_dir, name)) }.join(', ')
|
|
1118
|
+
abort "Consolidated report already exists: #{listed}\n" \
|
|
1119
|
+
"Rename or delete #{existing.size == 1 ? 'it' : 'all of them'} to consolidate again with the " \
|
|
1120
|
+
'same model configuration.'
|
|
1121
|
+
end
|
|
1122
|
+
prompt = File.read(prompt_path, encoding: ENCODING)
|
|
1123
|
+
abort "Prompt copy is empty: #{prompt_path}" if prompt.empty?
|
|
1124
|
+
|
|
1125
|
+
abort_unless_harness_installed
|
|
1126
|
+
|
|
1127
|
+
# In the same key order as a normal run's front matter.
|
|
1128
|
+
meta = fields.slice('role', 'timestamp', 'query', 'prompt_file', 'config_file', 'config_source',
|
|
1129
|
+
'run_stamp')
|
|
1130
|
+
meta['timestamp'] = iso_timestamp(meta['timestamp']) if meta['timestamp'].is_a?(Time)
|
|
1131
|
+
|
|
1132
|
+
puts "Config: #{relative_path(config_path)}"
|
|
1133
|
+
puts "Prompt: #{relative_path(prompt_path)}"
|
|
1134
|
+
puts "Run Dir: #{relative_path(run_dir)}"
|
|
1135
|
+
puts "\nConsolidator Agent:\n #{agent_descriptor(consolidator)}"
|
|
1136
|
+
puts "\nConsolidation Report (in #{relative_path(run_dir)}, " \
|
|
1137
|
+
"from #{reports.size} review#{'s' unless reports.size == 1}):"
|
|
1138
|
+
failures = missing_agents(config, reports)
|
|
1139
|
+
analysts, consolidations = read_run_file(run_dir)
|
|
1140
|
+
pool = WorkerPool.new(grace: INTERRUPT_GRACE_SECONDS)
|
|
1141
|
+
pool.trap_signals
|
|
1142
|
+
pool.start do
|
|
1143
|
+
timed do
|
|
1144
|
+
run_consolidation(consolidator, prompt, run_dir, file_prefix,
|
|
1145
|
+
meta: meta, prompt_path: relative_path(prompt_path), failures: failures,
|
|
1146
|
+
analyst_runs: analysts || [], pool: pool)
|
|
1147
|
+
end
|
|
1148
|
+
end
|
|
1149
|
+
(label, ok, detail, usage, meta_name), elapsed = pool.collect.first
|
|
1150
|
+
puts format_result_table([[label, ok, detail, elapsed, usage]])
|
|
1151
|
+
# Added to the run's run.json. Without one (deleted, or unreadable),
|
|
1152
|
+
# none is written: a total covering only this consolidation would
|
|
1153
|
+
# pass for the run's.
|
|
1154
|
+
if analysts
|
|
1155
|
+
entry = run_entry(consolidator, ok, detail, elapsed, usage, meta_report: meta_name)
|
|
1156
|
+
write_run_file(run_dir, analysts, consolidations + [entry])
|
|
1157
|
+
else
|
|
1158
|
+
warn "No readable #{RUN_FILENAME} in #{relative_path(run_dir)}; not recording this consolidation's run"
|
|
1159
|
+
end
|
|
1160
|
+
puts_consolidation_status(ok)
|
|
1161
|
+
exit(ok ? 0 : 1)
|
|
1162
|
+
rescue Interrupt
|
|
1163
|
+
exit_interrupted(pool)
|
|
1164
|
+
end
|
|
1165
|
+
|
|
1166
|
+
# True if name is a plain file name: no directory part or null byte, and
|
|
1167
|
+
# not empty, "." or "..".
|
|
1168
|
+
# Applied to every front matter value that becomes part of a path, so a
|
|
1169
|
+
# crafted report can't make --consolidate read or write outside the run
|
|
1170
|
+
# dir.
|
|
1171
|
+
module_function def bare_name?(name)
|
|
1172
|
+
name.is_a?(String) && !name.include?("\0") && !['', '.', '..'].include?(name) &&
|
|
1173
|
+
File.basename(name) == name
|
|
1174
|
+
end
|
|
1175
|
+
|
|
1176
|
+
# What --consolidate tells the consolidator about an agent without a report.
|
|
1177
|
+
MISSING_REPORT_REASON =
|
|
1178
|
+
'reason not recorded (it may have failed, been interrupted, or had its report removed)'
|
|
1179
|
+
|
|
1180
|
+
# For --consolidate: the config copy's agents that have no report, as
|
|
1181
|
+
# [label, reason] pairs. Why wasn't recorded, and needn't be a failure:
|
|
1182
|
+
# the run may have been interrupted, or a report removed.
|
|
1183
|
+
module_function def missing_agents(config, reports)
|
|
1184
|
+
agents = config['agents'].is_a?(Array) ? config['agents'] : []
|
|
1185
|
+
present = reports.map(&:first)
|
|
1186
|
+
agents.each_with_index.filter_map do |agent, i|
|
|
1187
|
+
next unless valid_agent_entry?(agent)
|
|
1188
|
+
|
|
1189
|
+
label = agent_label(agent['provider'], agent['model'], i + 1)
|
|
1190
|
+
[label, MISSING_REPORT_REASON] unless present.include?(label)
|
|
1191
|
+
end
|
|
1192
|
+
end
|
|
1193
|
+
|
|
1194
|
+
# Checked up front, before any run dir is created or model called.
|
|
1195
|
+
module_function def abort_unless_harness_installed
|
|
1196
|
+
return if executable_on_path?(HARNESS::NAME)
|
|
1197
|
+
|
|
1198
|
+
abort <<~MESSAGE
|
|
1199
|
+
#{HARNESS::NAME} executable not found; is `#{HARNESS::NAME}` installed and on your PATH?
|
|
1200
|
+
See "Prerequisites" in #{File.join(PROJECT_ROOT, 'README.md')}
|
|
1201
|
+
MESSAGE
|
|
1202
|
+
end
|
|
1203
|
+
|
|
1204
|
+
# After Ctrl-C (or another signal WorkerPool#trap_signals traps):
|
|
1205
|
+
# stops pool's agents, if any, and dies by the signal.
|
|
1206
|
+
module_function def exit_interrupted(pool)
|
|
1207
|
+
report = -> do
|
|
1208
|
+
warn "\nInterrupted."
|
|
1209
|
+
rescue IOError, SystemCallError
|
|
1210
|
+
nil # e.g. the terminal hung up
|
|
1211
|
+
end
|
|
1212
|
+
if pool
|
|
1213
|
+
pool.shutdown(&report)
|
|
1214
|
+
pool.exit_by_signal
|
|
1215
|
+
else # before any agent started; only a Ctrl-C can get here
|
|
1216
|
+
report.call
|
|
1217
|
+
WorkerPool.exit_by_signal('INT')
|
|
1218
|
+
end
|
|
1219
|
+
end
|
|
1220
|
+
end
|