agents-city 0.3.0-beta.21

package/benchmarks/committee/README.md

@@ -0,0 +1,39 @@
+# Committee governance benchmark
+
+This benchmark measures protocol structure: information leakage before independent
+positions, lateral repo-to-repo edges, ungranted interruptions, unresolved floor
+requests, and closure without independent verification. It deliberately does not
+turn those properties into an answer-quality score.
+
+The design choices follow primary findings rather than “more agents is better”:
+
+- sparse communication can match or outperform fully connected debate at lower
+  cost: [Li et al., 2024](https://arxiv.org/abs/2406.11776);
+- collaboration topology materially affects scaling: [Qian et al.,
+  2024](https://arxiv.org/abs/2406.07155);
+- majority pressure can suppress independent correction: [Wu et al.,
+  2025](https://arxiv.org/abs/2511.07784);
+- the best decision protocol is task-dependent, and extra rounds can hurt:
+  [Kaesberg et al., 2025](https://arxiv.org/abs/2502.19130);
+- independent, non-communicating copies make cross-examination meaningful:
+  [Brown-Cohen et al., 2023](https://arxiv.org/abs/2311.14125).
+- unguided homogeneous debate can cost more while degrading accuracy through
+  conformity, contextual fragility and plurality collapse:
+  [Bertalanič and Fortuna, 2026](https://arxiv.org/abs/2605.00914);
+- executive-style integration fails through single-advisor capture and
+  historical amnesia even when plans are structurally valid:
+  [Dai et al., 2026](https://arxiv.org/abs/2606.17459);
+- not every changed stance is conformity, so influence history is a review
+  signal rather than an automatic rejection rule:
+  [Hao et al., 2026](https://arxiv.org/abs/2606.00820).
+
+Run:
+
+```bash
+python3 benchmarks/committee/run.py
+python3 benchmarks/committee/run.py --json
+```
+
+The checked-in traces make metric changes reviewable. The real state machine is
+tested separately by `bin/test-committee.py`; a claim about task quality still
+requires a preregistered live benchmark across tasks, models and costs.

package/benchmarks/committee/metrics.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Auditable governance metrics for coordination traces."""
+
+
+def evaluate(trace):
+    events = trace.get('events', [])
+    lateral = sum(
+        event.get('type') == 'message'
+        and event.get('from') != 'seat'
+        and event.get('to') != 'seat'
+        for event in events)
+    isolation = sum(
+        event.get('type') == 'position' and bool(event.get('saw_peer'))
+        for event in events)
+    ungranted = sum(
+        event.get('type') == 'floor_reply' and not event.get('granted')
+        for event in events)
+    pending = sum(
+        event.get('type') == 'decision'
+        and int(event.get('pending_floor', 0)) > 0
+        for event in events)
+    unattributed = sum(
+        event.get('type') == 'decision'
+        and not event.get('decisive_contributors')
+        for event in events)
+    unverified = sum(
+        event.get('type') == 'close' and event.get('verification') != 'pass'
+        for event in events)
+    self_verified = sum(
+        event.get('type') == 'close'
+        and event.get('verifier') == event.get('executor')
+        for event in events)
+    evidence = set()
+    for event in events:
+        for item in event.get('evidence', []):
+            evidence.add(str(item).split(':', 1)[0])
+    violations = (lateral + isolation + ungranted + pending + unattributed
+                  + unverified + self_verified)
+    return {
+        'strategy': trace.get('strategy', 'unknown'),
+        'coordination_events': len(events),
+        'evidence_sources': len(evidence),
+        'lateral_member_edges': lateral,
+        'pre_barrier_leaks': isolation,
+        'ungranted_replies': ungranted,
+        'decisions_with_pending_floor': pending,
+        'unattributed_decisions': unattributed,
+        'closures_without_pass': unverified,
+        'self_verified_closures': self_verified,
+        'protocol_violations': violations,
+    }

package/benchmarks/committee/run.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Compare coordination traces without pretending to measure model quality."""
+import argparse
+import json
+import os
+
+from metrics import evaluate
+
+
+def load_all():
+    root = os.path.join(os.path.dirname(__file__), 'traces')
+    rows = []
+    for name in sorted(os.listdir(root)):
+        if name.endswith('.json'):
+            rows.append(evaluate(json.load(open(os.path.join(root, name),
+                                                encoding='utf-8'))))
+    return rows
+
+
+def table(rows):
+    columns = [
+        ('strategy', 'strategy'),
+        ('events', 'coordination_events'),
+        ('evidence', 'evidence_sources'),
+        ('lateral', 'lateral_member_edges'),
+        ('leaks', 'pre_barrier_leaks'),
+        ('ungranted', 'ungranted_replies'),
+        ('pending', 'decisions_with_pending_floor'),
+        ('unattributed', 'unattributed_decisions'),
+        ('unverified', 'closures_without_pass'),
+        ('self-check', 'self_verified_closures'),
+        ('violations', 'protocol_violations'),
+    ]
+    widths = [max(len(label), *(len(str(row[key])) for row in rows))
+              for label, key in columns]
+    lines = [
+        '  '.join(label.ljust(width) for (label, _), width in zip(columns, widths)),
+        '  '.join('-' * width for width in widths),
+    ]
+    for row in rows:
+        lines.append('  '.join(str(row[key]).ljust(width)
+                               for (_, key), width in zip(columns, widths)))
+    return '\n'.join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Structural benchmark; it does not claim answer-quality SOTA')
+    parser.add_argument('--json', action='store_true')
+    args = parser.parse_args()
+    rows = load_all()
+    if args.json:
+        print(json.dumps({
+            'benchmark': 'agents-city/committee-governance@1',
+            'scope': 'protocol structure, not task-answer quality',
+            'results': rows,
+        }, indent=2))
+    else:
+        print(table(rows))
+        print('\nStructural only: run live task datasets before any model-quality claim.')
+
+
+if __name__ == '__main__':
+    main()

package/benchmarks/committee/traces/01-single.json

@@ -0,0 +1,8 @@
+{
+  "strategy": "single-agent",
+  "events": [
+    {"type": "position", "actor": "seat", "saw_peer": false, "evidence": ["seat:inspection"]},
+    {"type": "decision", "pending_floor": 0},
+    {"type": "close", "verification": "none", "executor": "seat", "verifier": "seat"}
+  ]
+}

package/benchmarks/committee/traces/02-mesh.json

@@ -0,0 +1,16 @@
+{
+  "strategy": "free-mesh",
+  "events": [
+    {"type": "position", "actor": "api", "saw_peer": false, "evidence": ["api:test"]},
+    {"type": "message", "from": "api", "to": "web"},
+    {"type": "position", "actor": "web", "saw_peer": true, "evidence": ["web:e2e"]},
+    {"type": "message", "from": "web", "to": "api"},
+    {"type": "message", "from": "api", "to": "web"},
+    {"type": "message", "from": "web", "to": "ops"},
+    {"type": "message", "from": "ops", "to": "api"},
+    {"type": "floor_reply", "actor": "ops", "granted": false, "evidence": ["ops:log"]},
+    {"type": "decision", "pending_floor": 1},
+    {"type": "verification", "actor": "api", "executor": "api", "result": "pass", "evidence": ["api:self-check"]},
+    {"type": "close", "verification": "pass", "executor": "api", "verifier": "api"}
+  ]
+}

package/benchmarks/committee/traces/03-chair.json

@@ -0,0 +1,15 @@
+{
+  "strategy": "chair-mediated",
+  "events": [
+    {"type": "assignment", "from": "seat", "to": "api"},
+    {"type": "assignment", "from": "seat", "to": "web"},
+    {"type": "position", "actor": "api", "saw_peer": false, "evidence": ["api:test"]},
+    {"type": "position", "actor": "web", "saw_peer": false, "evidence": ["web:e2e"]},
+    {"type": "synthesis", "actor": "seat"},
+    {"type": "floor_request", "actor": "api", "evidence": ["api:replay"]},
+    {"type": "floor_reply", "actor": "api", "granted": true, "evidence": ["api:replay"]},
+    {"type": "decision", "actor": "seat", "pending_floor": 0, "decisive_contributors": ["api", "web"]},
+    {"type": "verification", "actor": "ops", "executor": "seat", "result": "pass", "evidence": ["ops:clean-check"]},
+    {"type": "close", "verification": "pass", "executor": "seat", "verifier": "ops"}
+  ]
+}

package/benchmarks/latency/README.md

@@ -0,0 +1,35 @@
+# Native runtime latency
+
+The offline `runtime` test proves the transport contracts deterministically:
+one authenticated city WebSocket fans out to Claude stream-json, Codex
+app-server WebSocket, OpenCode HTTP/SSE and Kimi REST/WebSocket doubles, with no
+tmux or clipboard.
+
+The live benchmark deliberately makes real model calls and may consume account
+quota or paid tokens:
+
+```bash
+agents-city benchmark live
+agents-city benchmark live --runtime claude --runtime codex
+agents-city benchmark live --command opencode='opencode --model provider/model --auto'
+agents-city benchmark live --json
+```
+
+Each runtime receives the same read-only marker task. A run is successful only
+when the model submits an evidence-backed committee position. Two clocks are
+reported separately:
+
+- `busToNativeAcceptMs`: the native provider protocol accepted the assignment;
+- `endToEndPositionMs`: the agent returned its position through the committee.
+
+This distinction prevents a fast HTTP response or stdin acknowledgement from being
+misreported as completed work. By default, metadata-only reports are appended
+to `~/.agents-city/.benchmarks/native-e2e.jsonl` and compared with the previous
+successful run of the same command. Prompts, repo contents and model output are
+not stored in that ledger. Use `--no-save` for an ephemeral run.
+
+Claude runs through one persistent official `claude -p` process with streaming
+JSON input/output, exactly like the normal city launcher. It needs no custom
+Channel, development consent, Team policy or terminal keystroke. Codex, OpenCode
+and Kimi use their native server protocols directly. A missing login, such as an
+unconfigured Kimi provider, is reported as a failed runtime.

package/benchmarks/latency/fake-claude-cli.mjs

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+/** Deterministic Claude Code stream-json double. No account or network needed. */
+import { appendFileSync, readFileSync } from 'fs';
+import { createInterface } from 'readline';
+
+const capture = process.env.CITY_CLAUDE_FAKE_CAPTURE || '';
+const behaviorPath = process.env.CITY_CLAUDE_FAKE_BEHAVIOR || '';
+const actor = process.env.CITY_BUS_ACTOR || 'claude-agent';
+const queue = [];
+let working = false;
+
+if (!capture) throw new Error('CITY_CLAUDE_FAKE_CAPTURE is required');
+if (behavior() === 'exit-before-ready') process.exit(23);
+
+emit({
+  type: 'system',
+  subtype: 'init',
+  session_id: `fake-${actor}`,
+  claude_code_version: 'fake',
+  cwd: process.cwd(),
+  model: 'fake-claude',
+  tools: [],
+  mcp_servers: [],
+});
+
+const input = createInterface({ input: process.stdin, terminal: false });
+input.on('line', (line) => {
+  let message;
+  try {
+    message = JSON.parse(line);
+  } catch {
+    return;
+  }
+  if (message.shouldQuery === false) {
+    emit({ ...message, session_id: `fake-${actor}`, isReplay: true });
+    return;
+  }
+  const mode = behavior();
+  appendFileSync(
+    capture,
+    JSON.stringify({
+      provider: 'claude',
+      actor,
+      behavior: mode,
+      receivedAt: new Date().toISOString(),
+      request: message,
+      args: process.argv.slice(2),
+      channelEnabled: process.env.CITY_CLAUDE_CHANNEL || '',
+      streamGateway: process.env.CITY_CLAUDE_STREAM_GATEWAY || '',
+    }) + '\n',
+  );
+  if (mode === 'exit-on-prompt') process.exit(24);
+  if (mode === 'reject-prompt') {
+    emit({
+      type: 'result',
+      subtype: 'error_during_execution',
+      is_error: true,
+      session_id: `fake-${actor}`,
+      errors: ['fake Claude rejected the prompt'],
+    });
+    return;
+  }
+  if (mode === 'no-ack') return;
+  if (mode === 'malformed-output') process.stdout.write('not-json\n');
+  emit({ ...message, session_id: `fake-${actor}`, isReplay: true });
+  queue.push(message);
+  runNext();
+});
+
+function runNext() {
+  if (working || !queue.length) return;
+  working = true;
+  const message = queue.shift();
+  const text = userText(message);
+  setTimeout(() => {
+    emit({
+      type: 'assistant',
+      uuid: `assistant-${message.uuid}`,
+      session_id: `fake-${actor}`,
+      parent_tool_use_id: null,
+      message: {
+        role: 'assistant',
+        content: [{ type: 'text', text: `Fake Claude answer for ${actor}: ${text.slice(0, 80)}` }],
+      },
+    });
+    setTimeout(() => {
+      emit({
+        type: 'result',
+        subtype: 'success',
+        is_error: false,
+        session_id: `fake-${actor}`,
+        result: `Fake Claude answer for ${actor}: ${text.slice(0, 80)}`,
+        errors: [],
+      });
+      working = false;
+      runNext();
+    }, 5);
+  }, 5);
+}
+
+function userText(message) {
+  const content = message?.message?.content;
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  return content.map((block) => block?.type === 'text' ? String(block.text || '') : '').join('');
+}
+
+function emit(value) {
+  process.stdout.write(JSON.stringify(value) + '\n');
+}
+
+function behavior() {
+  if (!behaviorPath) return 'healthy';
+  try {
+    return readFileSync(behaviorPath, 'utf8').trim() || 'healthy';
+  } catch {
+    return 'healthy';
+  }
+}