@piagent/platform 1.36.1 → 1.37.1

package/evals/long-task-soak-v1/scenarios/long-python-billing-platform.json

@@ -0,0 +1,84 @@
+{
+  "id": "long-python-billing-platform",
+  "title": "A long session building a billing platform in six phases (v1, dunning, multi-currency, a 3,500-line legacy refactor, SQLite import of 15,000 records, CLI) with a rule change, two Stops and a Gateway restart along the way",
+  "category": ["long-horizon", "business", "python", "mixed"],
+  "language": "en",
+  "fixture": "python-billing-long",
+  "requires": ["uv", "network"],
+  "prompt": "We have a long day on the billing package. First piece: please implement the whole of SPEC.md: money rounding, coupons, tax, invoices, subscriptions and billing cycles, mid-cycle plan changes, refunds and the CSV/JSON export, all under src/billing/. Add tests for every section under tests/ and run `.venv/bin/python -m pytest -q` until everything passes. A few business rules in the spec are still marked TBD; I'm the product owner, so ask me about those, and don't guess. Everything else in the spec is decided, so carry on through to the end without stopping. The documents under docs/ are the next pieces of work; I'll bring them up one at a time.",
+  "phases": [
+    {
+      "id": "p2",
+      "prompt": "Thanks, v1 is in. Next piece: implement docs/DUNNING.md, failed payments as a state machine with the retry schedule, the grace period and suspension, with tests for every rule. One rule in there is still open and only I can decide it, so ask me; don't guess. Everything from before must keep passing (`.venv/bin/python -m pytest -q`). Carry on to the end without stopping.",
+      "oracle": { "hidden": "hidden-p2", "checks": [{ "name": "dunning acceptance with the product owner's answer, plus everything before", "run": ".venv/bin/python -m pytest -q", "timeoutSec": 600 }] },
+      "reference": "reference-p2"
+    },
+    {
+      "id": "p3",
+      "prompt": "Good. Next: multi-currency billing as described in docs/MULTICURRENCY.md, with the exchange rates in data/fx_rates.csv (don't edit that file). One business rule there is still open; ask me about it before you implement it. Add tests for every rule, keep the whole suite green and carry on to the end.",
+      "oracle": { "hidden": "hidden-p3", "checks": [{ "name": "multi-currency acceptance with the product owner's answer, plus everything before", "run": ".venv/bin/python -m pytest -q", "timeoutSec": 600 }] },
+      "reference": "reference-p3"
+    },
+    {
+      "id": "p4",
+      "prompt": "Next is the big one. src/billing/legacy_reports.py (about 3,500 lines) has to become a proper package, as planned in docs/REPORTS.md: split by topic, the shared logic written once, full type hints, and every report byte-for-byte identical to the goldens in tests/golden/reports. Finance diff these files, so not a single character of output may change. Read the legacy code carefully before you cut it up. Keep the full suite green and carry on until it's done.",
+      "oracle": { "hidden": "hidden-p4", "checks": [{ "name": "reports package: golden outputs, structure, type hints, plus everything before", "run": ".venv/bin/python -m pytest -q", "timeoutSec": 600 }] },
+      "reference": "reference-p4"
+    },
+    {
+      "id": "p5",
+      "prompt": "Next: persistence per docs/PERSISTENCE.md: the SQLite repository, schema migrations v1 to v2 including the data migration, and the import of data/invoices_2025.jsonl with its error report. The old export has some bad lines; the import must report them, not choke on them, and don't edit the data file. Everything is decided in that document. Tests for every rule, full suite green, carry on to the end.",
+      "oracle": { "hidden": "hidden-p5", "checks": [{ "name": "storage, migrations and the 2025 import, plus everything before", "run": ".venv/bin/python -m pytest -q", "timeoutSec": 600 }] },
+      "reference": "reference-p5"
+    },
+    {
+      "id": "p6",
+      "prompt": "Last piece for today: the `python -m billing` command line from docs/CLI.md (migrate, import, report with every report name, export), documented in README.md with examples. Then do a final regression pass over everything we did today (SPEC.md, the dunning, currency, reports, persistence and CLI documents, and the rule changes I gave you along the way) and make sure the whole suite passes.",
+      "oracle": { "hidden": "hidden-p6", "checks": [{ "name": "CLI, README, the invoice-number rule change, and a full regression", "run": ".venv/bin/python -m pytest -q", "timeoutSec": 900 }] },
+      "reference": "reference-p6"
+    }
+  ],
+  "questionsExpected": false,
+  "decisions": [
+    {
+      "id": "suspension-access", "mustAsk": true,
+      "topic": ["suspen", "(lose|loses|losing|keep|keeps|keeping|retain|retains) access", "access.{0,40}(immediate|right away|period end|end of)", "đình chỉ", "tạm ngưng", "khóa (tài khoản|dịch vụ)", "mất quyền", "quyền truy cập"],
+      "option": "^\\W*(access\\s+)?(ends?\\s+|stops?\\s+|lost\\s+|revoked\\s+|cut( off)?\\s+)?immediate|^\\W*(revoke|cut|end|block|stop|remove|lose)\\w*\\b[^.]{0,40}\\b(immediately|at once|right away|on suspension)|^\\W*(ngay|chấm dứt ngay|khóa ngay|mất quyền)",
+      "reply": "When a subscription is suspended the customer loses access immediately, from the suspension date on. They do not keep access until the end of the period they already paid for. Paying the overdue invoice gives access back."
+    },
+    {
+      "id": "refund-fx-date", "mustAsk": true,
+      "topic": ["(fx|exchange|currency|conversion).{0,60}refund", "refund.{0,60}(fx|exchange|currency|conver)", "tỷ giá", "quy đổi"],
+      "option": "^\\W*(use\\s+)?(the\\s+)?(original|invoice|issue)|^\\W*(rate|fx rate|exchange rate)\\s+(at|on|of)\\s+(the\\s+)?(original|invoice|issue)|^\\W*(tỷ giá\\s+)?(ngày\\s+)?(gốc|hóa đơn|xuất hóa đơn)",
+      "reply": "Convert a refund at the rate of the original invoice date (the invoice's fx_rate), not at the rate on the refund date, so a full refund cancels the invoice exactly in the USD books."
+    },
+    {
+      "id": "upgrade-charge", "mustAsk": true,
+      "topic": ["prorat", "upgrade", "mid.?cycle", "plan change", "change.{0,10}plan", "giữa kỳ", "nâng cấp", "đổi gói", "tính theo ngày"],
+      "option": "full (new )?month|full monthly|full.{0,20}new plan|whole month|cả tháng|nguyên tháng",
+      "reply": "On an upgrade, charge the full monthly price of the new plan immediately: no proration and no credit for the old plan. Taxed as usual, no coupon."
+    },
+    {
+      "id": "rounding", "mustAsk": true,
+      "topic": ["round", "half.?(up|even)", "banker", "làm tròn"],
+      "option": "half.?even|banker|chẵn",
+      "reply": "Round half-even (banker's rounding) to 2 decimal places, everywhere we round."
+    },
+    {
+      "id": "coupon-vs-tax", "mustAsk": true,
+      "topic": ["coupon", "discount", "(before|after).{0,15}tax", "tax.{0,20}(before|after|coupon|discount)", "giảm giá", "khuyến mãi", "mã giảm", "trước thuế", "sau thuế"],
+      "option": "^\\W*(apply\\s+|(the\\s+)?(coupons?|discounts?)\\s+)*after.{0,10}tax|post.?tax|^\\W*sau thuế",
+      "reply": "Apply the coupon after tax: compute tax on the full subtotal, then take the discount from subtotal + tax."
+    }
+  ],
+  "approvals": "allow",
+  "userActions": [
+    { "when": { "minutes": 10 }, "do": "steer", "text": "One rule change from finance, please fold it in alongside whatever you are doing: invoice numbers must follow the format INV-YYYY-NNNNN, where YYYY is the year of the invoice's issue date and NNNNN is a five-digit sequence in issue order, per service, that restarts at 00001 every calendar year (INV-2026-00001, INV-2026-00002, ..., INV-2027-00001). Credit notes keep CN-0001. Update the code, the tests and SPEC.md §6." },
+    { "when": { "minutes": 28 }, "do": "stop_then_continue", "text": "Sorry, I pressed Stop by accident. Please carry on from where you were." },
+    { "when": { "minutes": 48 }, "do": "restart", "hard": true, "text": "Piagent just updated and restarted, please carry on." },
+    { "when": { "minutes": 70 }, "do": "stop_then_continue", "text": "I stopped you to check something; all good. Please continue where you left off." }
+  ],
+  "budget": { "maxMinutes": 150, "maxNudges": 4, "stallMinutes": 15 },
+  "oracle": { "hidden": "hidden-main", "checks": [{ "name": "SPEC v1 acceptance with the product owner's answers", "run": ".venv/bin/python -m pytest -q", "timeoutSec": 600 }] },
+  "reference": "reference-main"
+}

package/evals/long-task-soak-v1/scenarios/mixed-requirement-change.json

@@ -0,0 +1,20 @@
+{
+  "id": "mixed-requirement-change",
+  "title": "The member changes a rule mid-task, presses Stop, and reopens the conversation",
+  "category": ["mixed", "interruption", "long-horizon"],
+  "language": "vi",
+  "fixture": "node-inventory",
+  "requires": [],
+  "prompt": "Cài đặt đầy đủ SPEC.md của kho-hang (src/ và test/), chạy npm test đến khi qua hết.",
+  "questionsExpected": false,
+  "decisions": [],
+  "approvals": "allow",
+  "userActions": [
+    { "when": { "toolCalls": 12 }, "do": "steer", "text": "À anh đổi một quy tắc: lowStock(threshold) phải tính cả SKU có available đúng bằng threshold (<=, không phải <). Cập nhật code, test và SPEC.md cho khớp nhé." },
+    { "when": { "toolCalls": 25 }, "do": "stop_then_continue", "text": "Anh lỡ bấm dừng, em làm tiếp phần còn lại nhé." },
+    { "when": { "toolCalls": 40 }, "do": "restart", "text": "Anh vừa mở lại cửa sổ. Làm tiếp cho xong nhé." }
+  ],
+  "budget": { "maxMinutes": 75, "maxNudges": 3, "stallMinutes": 12 },
+  "oracle": { "hidden": "hidden-change", "checks": [{ "name": "spec acceptance with the changed rule", "run": "node --test 'test/*.test.js'", "timeoutSec": 300 }] },
+  "reference": "reference-change"
+}

package/evals/long-task-soak-v1/scenarios/stability-flaky-slow-suite.json

@@ -0,0 +1,22 @@
+{
+  "id": "stability-flaky-slow-suite",
+  "title": "Fix a flaky test at its root in a suite that takes ~2.7 minutes, and prove it with a deterministic regression test",
+  "category": ["stability", "long-running-commands", "autonomous"],
+  "language": "en",
+  "fixture": "flaky-slow-suite",
+  "requires": [],
+  "prompt": "Our CI keeps flaking on \"withRetry gives up before crossing the deadline (real timers)\" in ingest-relay; details are in ISSUE.md and the contract is docs/retry.md. Please find the root cause in src/ and fix it properly (no retries, skips or loosened assertions), add the `random` option, write a deterministic regression test that would fail on the old code, and run the whole suite green at least twice. Heads up: npm test takes about 3 minutes on purpose, let it finish. No need to check back with me, just get it done.",
+  "questionsExpected": false,
+  "decisions": [],
+  "approvals": "allow",
+  "budget": { "maxMinutes": 60, "maxNudges": 3, "stallMinutes": 10 },
+  "oracle": {
+    "hidden": "hidden",
+    "checks": [
+      { "name": "no skip/todo/only or retry wrappers, soak suite and real-timer test kept, regression test injects random", "run": "node .soak/check_static.mjs", "timeoutSec": 60, "stopOnFail": false },
+      { "name": "deterministic retry contract: injected random, one draw per retry, never past the deadline (seeded sweep)", "run": "node --test .soak/retry-regression.hidden.test.mjs", "timeoutSec": 120 },
+      { "name": "full suite once (about 2.7 minutes of real timers)", "run": "node --test 'test/*.test.js'", "timeoutSec": 600 }
+    ]
+  },
+  "reference": "reference"
+}

package/evals/long-task-soak-v1/scenarios/thirdparty-postgres-integration.json

@@ -0,0 +1,30 @@
+{
+  "id": "thirdparty-postgres-integration",
+  "title": "Meeting-room booking service on real PostgreSQL: migrations, concurrency-safe overlap checks, soft cancel, keyset pagination",
+  "category": ["third-party", "business", "integration", "node"],
+  "language": "vi",
+  "fixture": "node-postgres",
+  "requires": ["docker", "network"],
+  "prompt": "Em làm giúp anh toàn bộ SPEC.md cho dịch vụ đặt lịch phòng họp nhé: thêm migration mới, hoàn thiện repository (phòng họp và đặt lịch), chống trùng lịch an toàn khi nhiều người đặt cùng lúc, huỷ đặt lịch, danh sách có phân trang, và viết test tích hợp chạy với PostgreSQL thật (dùng scripts/db-up.sh hoặc docker-compose.yml). Chỗ nào SPEC ghi \"cần chủ sản phẩm chốt\" thì hỏi anh trước khi làm phần đó, còn lại em cứ tự làm đến khi npm run migrate && npm test qua hết. Xong thì dừng container Postgres em đã bật.",
+  "questionsExpected": false,
+  "decisions": [
+    {
+      "id": "cancel-soft-delete",
+      "mustAsk": true,
+      "topic": ["hu[ỷy]", "h[ủu]y", "cancel", "x[oó][aáà]", "delete", "soft", "xoá mềm|xóa mềm|xoá hẳn|xóa hẳn"],
+      "option": "soft|mềm|cancelled_at|giữ lại|đánh dấu|mark",
+      "reply": "Huỷ là xoá mềm: giữ lại bản ghi và đặt cancelled_at = thời điểm huỷ (cột cancelled_at trong bảng bookings). Đặt lịch đã huỷ không tính khi kiểm tra trùng và không hiện trong listBookings mặc định, nhưng listBookings({ includeCancelled: true }) vẫn phải trả về chúng (kèm cancelledAt), getBooking vẫn trả về được, và huỷ lần nữa thì báo ALREADY_CANCELLED."
+    },
+    {
+      "id": "back-to-back",
+      "mustAsk": true,
+      "topic": ["li[eề]n k[eề]", "liền nhau|nối tiếp|sát nhau", "back.?to.?back", "adjacent", "end.*start", "kết thúc.*bắt đầu", "ch[aạ]m", "\\[\\s*\\)|half.?open|nửa mở"],
+      "option": "không (bị |được )?(tính là |coi là )?(trùng|xung đột)|cho phép|allow|not (a )?conflict|no conflict|don.?t conflict|half.?open|\\[\\)",
+      "reply": "Hai lịch liền kề (giờ kết thúc lịch này bằng giờ bắt đầu lịch kia, ví dụ 09:00–10:00 và 10:00–11:00) KHÔNG tính là trùng, phải cho phép đặt. Chỉ khi hai khoảng thật sự giao nhau mới là BOOKING_CONFLICT."
+    }
+  ],
+  "approvals": "allow",
+  "budget": { "maxMinutes": 75, "maxNudges": 3, "stallMinutes": 15 },
+  "oracle": { "hidden": "hidden", "checks": [{ "name": "spec acceptance on a throwaway PostgreSQL", "run": "bash scripts/oracle.sh", "timeoutSec": 600 }] },
+  "reference": "reference"
+}

package/evals/long-task-soak-v1/surfaces/terminal-rpc.mjs

@@ -0,0 +1,150 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawn, execFileSync } from 'node:child_process';
+import { piagentEnv } from '../lib/workspace.mjs';
+
+// The company Terminal (`piagent studio --rpc`) driven as the member: one
+// child process per conversation, Pi's RPC protocol on stdin/stdout.
+const SKIP = 'Để agent tự quyết';
+const OTHER = /^\d+\. Khác \(tự nhập\)$/;
+const ALLOW = /^(?:allow|cho phép|đồng ý|yes|có|chạy|run)/i, DENY = /^(?:block|deny|từ chối|no|không|chặn)/i;
+
+export function terminalSupportsRpc(piagent = 'piagent') {
+  try { return execFileSync(piagent, ['studio', '--help'], { encoding: 'utf8', env: piagentEnv() }).includes('--rpc'); }
+  catch { return false; }
+}
+
+// "(1/2) Header: question" and "1. Label — description" as ask_user shows them.
+function parseQuestion(title, options) {
+  const question = title.replace(/^\(\d+\/\d+\)\s*/, '').split('\nĐã chọn:')[0];
+  const real = options.filter(o => o !== SKIP && !OTHER.test(o) && o !== 'Xong');
+  return { question, options: real.map(o => { const [label, ...rest] = o.replace(/^\d+\.\s*/, '').split(' — '); return { label, description: rest.join(' — ') }; }), raw: real };
+}
+
+export class TerminalSurface {
+  constructor({ piagent = 'piagent' } = {}) { this.piagent = piagent; this.seq = 0; }
+
+  async open({ project, runDir, hooks }) {
+    Object.assign(this, { project, runDir, hooks });
+    this.stderrLog = fs.createWriteStream(path.join(runDir, 'terminal-stderr.log'), { flags: 'a' });
+    await this.#spawn();
+  }
+
+  async #spawn() {
+    const child = spawn(this.piagent, ['studio', '--rpc', '--project', this.project], { env: piagentEnv(), stdio: ['pipe', 'pipe', 'pipe'] });
+    this.child = child; this.pending = new Map(); this.closing = false; this.stderr = '';
+    let buffer = Buffer.alloc(0);
+    child.stdout.on('data', chunk => {
+      buffer = Buffer.concat([buffer, chunk]);
+      for (let i; (i = buffer.indexOf(10)) >= 0;) {
+        const line = buffer.subarray(0, i).toString('utf8').replace(/\r$/, ''); buffer = buffer.subarray(i + 1);
+        if (!line.trim()) continue;
+        let record; try { record = JSON.parse(line); } catch { continue; }
+        this.#onRecord(record);
+      }
+    });
+    child.stderr.on('data', d => { this.stderr = (this.stderr + d).slice(-4000); this.stderrLog.write(d); });
+    child.once('exit', (code, signal) => {
+      for (const p of this.pending.values()) p.reject(Error(`terminal exited ${code ?? signal}`));
+      this.pending.clear();
+      if (!this.closing && this.turn) this.#endTurn({ crashed: true, error: `exit ${code ?? signal}: ${this.stderr.slice(-800)}` });
+    });
+    child.stdin.on('error', () => { /* the exit handler reports it */ });
+    await this.#command({ type: 'get_state' }, 180_000).then(s => { this.sessionFile = s?.sessionFile ?? this.sessionFile; });
+  }
+
+  #command(command, timeoutMs = 60_000) {
+    const id = `soak-${++this.seq}`;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => { this.pending.delete(id); reject(Error(`no response to ${command.type}`)); }, timeoutMs);
+      this.pending.set(id, { resolve: v => { clearTimeout(timer); resolve(v); }, reject: e => { clearTimeout(timer); reject(e); } });
+      this.child.stdin.write(JSON.stringify({ ...command, id }) + '\n');
+    });
+  }
+  #reply(record) { this.child.stdin.write(JSON.stringify(record) + '\n'); }
+
+  #onRecord(r) {
+    const hooks = this.hooks;
+    if (r.type === 'response') {
+      const p = this.pending.get(r.id); if (!p) return; this.pending.delete(r.id);
+      return r.success ? p.resolve(r.data) : p.reject(Error(`${r.command}: ${r.error}`));
+    }
+    if (r.type === 'extension_ui_request') return this.#onUi(r);
+    if (!this.turn) return;
+    if (r.type === 'message_update' && r.assistantMessageEvent?.type === 'text_delta') hooks.onEvent({ type: 'text_delta' });
+    else if (r.type === 'message_end' && r.message?.role === 'assistant') {
+      const m = r.message, text = (m.content ?? []).filter(c => c.type === 'text').map(c => c.text).join('\n');
+      if (text.trim()) this.turn.lastText = text;
+      if (m.stopReason === 'error') this.turn.error = m.errorMessage ?? 'error'; else if (text.trim()) this.turn.error = null;
+      hooks.onEvent({ type: 'assistant_message', stopReason: m.stopReason, inputTokens: m.usage?.input, outputTokens: m.usage?.output, error: m.errorMessage });
+    }
+    else if (r.type === 'message_end' && r.message?.role === 'custom') hooks.onEvent({ type: 'note', customType: r.message.customType, text: String(r.message.content ?? '').slice(0, 300) });
+    else if (r.type === 'tool_execution_start') hooks.onEvent({ type: 'tool_start', name: r.toolName });
+    else if (r.type === 'tool_execution_end') hooks.onEvent({ type: 'tool_end', name: r.toolName, isError: !!r.isError });
+    else if (r.type === 'tool_execution_update') hooks.onEvent({ type: 'text_delta' });
+    else if (r.type === 'auto_retry_start' || r.type === 'managed_provider_retry') hooks.onEvent({ type: 'provider_retry', code: r.code ?? r.errorMessage, attempt: r.attempt });
+    else if (r.type === 'compaction_start') hooks.onEvent({ type: 'compaction', reason: r.reason });
+    else if (r.type === 'managed_helpers') hooks.onEvent({ type: 'helpers', active: r.active });
+    else if (r.type === 'managed_turn_end') this.turn.turnEnd = r.end;
+    else if (r.type === 'agent_settled') this.#endTurn({});
+  }
+
+  async #onUi(r) {
+    const hooks = this.hooks;
+    if (r.method === 'notify') { hooks.onEvent({ type: 'notice', text: String(r.message ?? '').slice(0, 300), level: r.notifyType }); return; }
+    if (r.method === 'setStatus') { if (r.statusText) hooks.onEvent({ type: 'status', key: r.statusKey, text: String(r.statusText).slice(0, 200) }); return; }
+    if (!['select', 'confirm', 'input', 'editor'].includes(r.method)) return;
+    if (r.method === 'select' && r.options?.includes(SKIP)) {
+      const q = parseQuestion(r.title, r.options), answer = await hooks.onQuestion(q);
+      const other = r.options.find(o => OTHER.test(o));
+      const value = answer.skipped ? SKIP : answer.other != null ? other : q.raw[answer.selected[0]];
+      this.otherText = answer.other ?? null;
+      return this.#reply({ type: 'extension_ui_response', id: r.id, value });
+    }
+    if (r.method === 'input' && this.otherText != null) { const value = this.otherText; this.otherText = null; return this.#reply({ type: 'extension_ui_response', id: r.id, value }); }
+    if (r.method === 'confirm') return this.#reply({ type: 'extension_ui_response', id: r.id, confirmed: !!(await hooks.onApproval({ title: `${r.title} ${r.message ?? ''}` })) });
+    if (r.method === 'select') {
+      const allow = await hooks.onApproval({ title: `${r.title} [${r.options.join(' | ')}]` });
+      const value = r.options.find(o => (allow ? ALLOW : DENY).test(o));
+      return this.#reply(value ? { type: 'extension_ui_response', id: r.id, value } : { type: 'extension_ui_response', id: r.id, cancelled: true });
+    }
+    this.#reply({ type: 'extension_ui_response', id: r.id, cancelled: true });
+  }
+
+  #endTurn(extra) {
+    const turn = this.turn; if (!turn) return;
+    this.turn = null;
+    turn.resolve({ lastText: turn.lastText, turnEnd: turn.turnEnd, error: turn.error, ...extra });
+  }
+
+  async send(text, delivery = 'new') {
+    if (delivery !== 'new' && this.turn) return this.#command({ type: delivery === 'steer' ? 'steer' : 'follow_up', message: text });
+    let resolve; const ended = new Promise(r => { resolve = r; });
+    this.turn = { lastText: '', turnEnd: null, error: null, resolve, ended };
+    try { await this.#command({ type: 'prompt', message: text }); }
+    catch (error) { this.#endTurn({ error: String(error.message) }); }
+  }
+  nextTurnEnd() { return this.turn ? this.turn.ended : Promise.resolve({ lastText: '', turnEnd: null, error: 'no turn running' }); }
+  async stop() { if (this.child?.exitCode === null) await this.#command({ type: 'abort' }, 120_000); }
+
+  // The member closes the Terminal window mid-task and opens the same
+  // conversation again (/resume).
+  async restart() { // the Terminal closing is always hard: the conversation stops with it
+    if (!this.sessionFile) return false;
+    this.closing = true; this.child.kill('SIGKILL');
+    await new Promise(r => this.child.exitCode !== null || this.child.signalCode ? r() : this.child.once('exit', r));
+    this.#endTurn({ restarted: true });
+    await this.#spawn();
+    const switched = await this.#command({ type: 'switch_session', sessionPath: this.sessionFile }, 120_000);
+    return switched?.cancelled ? false : 'reopened';
+  }
+
+  async close() {
+    this.closing = true;
+    if (!this.child || this.child.exitCode !== null) return;
+    const exited = new Promise(r => this.child.once('exit', r));
+    this.child.stdin.end();
+    const timer = setTimeout(() => this.child.kill('SIGKILL'), 30_000);
+    await exited; clearTimeout(timer); this.stderrLog.end();
+  }
+}

package/evals/long-task-soak-v1/surfaces/webui-gateway.mjs

@@ -0,0 +1,214 @@
+import { spawn } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { WebSocket } from 'ws';
+import { piagentEnv } from '../lib/workspace.mjs';
+
+// The company WebUI (Agent Watch Auto) used the way the browser app uses it:
+// the launch link's capability for a cookie and CSRF token, the Gateway
+// WebSocket for commands and live events, HTTP for questions, approvals and
+// the transcript. No browser: what is measured is the agent, not the page.
+const opaque = prefix => `${prefix}_${randomUUID().replaceAll('-', '')}`;
+const PROTOCOL = { schemaVersion: 1, version: 'piagent-gateway-protocol-v1' };
+const wait = ms => new Promise(r => setTimeout(r, ms));
+
+export class WebUiSurface {
+  constructor({ piagent = 'piagent' } = {}) { this.piagent = piagent; this.pending = new Map(); this.answered = new Set(); }
+
+  // `--web` prints the launch link; when no company Gateway runs yet it starts
+  // one and keeps serving it, so read the link without waiting for an exit
+  // and keep that process until the run closes.
+  #launch() {
+    return new Promise((resolve, reject) => {
+      const child = spawn(this.piagent, ['studio', '--web', '--no-open', '--project', this.project], { env: piagentEnv(), stdio: ['ignore', 'pipe', 'pipe'] });
+      let out = '';
+      const timer = setTimeout(() => { child.kill(); reject(Error(`no launch link in 180 s: ${out.slice(-300)}`)); }, 180_000);
+      const read = d => {
+        out += d;
+        const url = out.match(/Agent Watch Auto: (http\S+)/)?.[1];
+        if (!url) return;
+        clearTimeout(timer); child.stdout.off('data', read);
+        if (child.exitCode === null) this.gatewayOwner ??= child;
+        const parsed = new URL(url), hash = new URLSearchParams(parsed.hash.slice(1));
+        resolve({ origin: parsed.origin, capability: hash.get('bootstrap'), projectRef: hash.get('project') });
+      };
+      child.stdout.on('data', read); child.stderr.on('data', d => { out += d; });
+      child.once('exit', code => { clearTimeout(timer); if (!out.includes('Agent Watch Auto: http')) reject(Error(`studio --web exited ${code}: ${out.slice(-400)}`)); });
+    });
+  }
+
+  async #login() {
+    // Right after another run's Gateway stopped, a launch can fail once
+    // (managed-launch-failed) while that Gateway is still going away.
+    let launched;
+    try { launched = await this.#launch(); } catch { await wait(15_000); launched = await this.#launch(); }
+    const { origin, capability, projectRef } = launched;
+    const response = await fetch(`${origin}/api/v1/bootstrap`, { method: 'POST', headers: { Origin: origin, 'Content-Type': 'application/json' }, body: JSON.stringify({ capability }) });
+    const cookie = response.headers.get('set-cookie')?.split(';', 1)[0], body = await response.json().catch(() => null);
+    if (!response.ok || !cookie || !body?.csrfToken) throw Error(`bootstrap failed ${response.status}`);
+    Object.assign(this, { origin, cookie, csrf: body.csrfToken, projectRef });
+  }
+
+  async http(method, path, body) {
+    for (let attempt = 0; ; attempt++) {
+      const response = await fetch(`${this.origin}${path}`, { method, headers: { Accept: 'application/json', Origin: this.origin, Cookie: this.cookie,
+        ...(method === 'POST' ? { 'Content-Type': 'application/json', 'X-Piagent-CSRF': this.csrf } : {}) }, body: body === undefined ? undefined : JSON.stringify(body) });
+      if (response.status === 401 && attempt === 0) { await this.#login(); continue; }
+      return { status: response.status, value: await response.json().catch(() => null) };
+    }
+  }
+
+  #connect() {
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(`${this.origin.replace('http:', 'ws:')}/api/v1/gateway`, 'piagent.gateway.v1', { headers: { Origin: this.origin, Cookie: this.cookie }, perMessageDeflate: false });
+      const timer = setTimeout(() => { ws.terminate(); reject(Error('gateway hello timeout')); }, 10_000);
+      ws.on('open', () => ws.send(JSON.stringify({ ...PROTOCOL, messageType: 'connect', clientRef: 'soak_member', minimumProtocol: 1, maximumProtocol: 1, lastEventSequence: null, catalogRevision: null })));
+      ws.on('message', data => {
+        let frame; try { frame = JSON.parse(String(data)); } catch { return; }
+        if (frame.messageType === 'hello') { clearTimeout(timer); resolve(ws); return; }
+        if (frame.messageType === 'response') {
+          const p = this.pending.get(frame.requestId); if (!p) return; this.pending.delete(frame.requestId);
+          return frame.ok ? p.resolve(frame.result) : p.reject(Error(String(frame.error?.code ?? 'gateway-request-failed')));
+        }
+        if (frame.messageType === 'event') this.#onEvent(frame.kind, frame.payload ?? {});
+      });
+      ws.on('close', () => { if (!this.closing) this.socketLost = true; });
+      ws.on('error', error => { clearTimeout(timer); reject(error); });
+    });
+  }
+
+  #request(command) {
+    const requestId = opaque('request');
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => { this.pending.delete(requestId); reject(Error('gateway command timeout')); }, 60_000);
+      this.pending.set(requestId, { resolve: v => { clearTimeout(timer); resolve(v); }, reject: e => { clearTimeout(timer); reject(e); } });
+      this.ws.send(JSON.stringify({ ...PROTOCOL, messageType: 'request', requestId, method: 'sessions.command', params: { command } }));
+    });
+  }
+
+  async #command(action, payload) {
+    for (let attempt = 0; attempt < 4; attempt++) {
+      const catalog = (await this.http('GET', '/api/v1/session-catalog')).value;
+      const row = this.sessionRef ? catalog?.sessions?.find(s => s.sessionRef === this.sessionRef) : null;
+      const now = new Date();
+      const receipt = await this.#request({ schemaVersion: 1, version: 'piagent-session-command-v1', messageType: 'command', commandId: opaque('command'), idempotencyKey: opaque('idempotency'),
+        action, requestedAt: now.toISOString(), expiresAt: new Date(now.getTime() + 5 * 60_000).toISOString(), sessionRef: action === 'session.create' ? null : this.sessionRef,
+        expectedCatalogRevision: catalog.catalogRevision, expectedSessionRevision: action === 'session.create' ? null : row?.sessionRevision ?? null, payload });
+      if (receipt?.resultCode === 'stale-revision') { await wait(500); continue; }
+      return receipt;
+    }
+    throw Error(`${action}: stale revision four times`);
+  }
+
+  async open({ project, hooks }) {
+    Object.assign(this, { project, hooks });
+    await this.#login();
+    this.ws = await this.#connect();
+    const options = (await this.http('GET', '/api/v1/session-creation-options')).value;
+    const place = options?.projects?.find(p => p.projectRef === this.projectRef);
+    if (!place) throw Error('project not offered for new conversations');
+    this.create = { projectRef: this.projectRef, placeRef: place.placeRef, modelRef: options.defaultModelRef ?? null, thinkingLevel: options.defaultThinkingLevel ?? 'medium' };
+    this.poller = setInterval(() => void this.#poll().catch(() => {}), 3000);
+  }
+
+  #onEvent(kind, p) {
+    if (!this.sessionRef || p.sessionRef !== this.sessionRef) return;
+    const hooks = this.hooks;
+    if (p.operationRef) this.operationRef = p.operationRef;
+    if (kind === 'message.delta') hooks.onEvent({ type: 'text_delta' });
+    else if (kind === 'tool.started') {
+      if (p.toolLabel === 'compaction') hooks.onEvent({ type: 'compaction' });
+      else if (p.toolLabel === 'capacity-wait') hooks.onEvent({ type: 'capacity_wait' });
+      else hooks.onEvent({ type: 'tool_start', name: p.toolLabel });
+    }
+    else if (kind === 'tool.completed') hooks.onEvent({ type: 'tool_end', name: p.toolLabel, isError: !!p.isError, reasonCode: p.reasonCode });
+    else if (kind === 'message.completed') hooks.onEvent({ type: 'assistant_message' });
+    else if (kind === 'runtime.changed') hooks.onEvent({ type: 'status', key: 'liveState', text: `${p.liveState} ${p.reasonCode ?? ''}`.trim() });
+    else if (kind === 'operation.settled') void this.#settled(p);
+  }
+
+  async #settled(p) {
+    if (!this.turn) return;
+    const page = (await this.http('GET', `/api/v1/sessions/${encodeURIComponent(this.sessionRef)}/inspection/transcript?limit=60`)).value;
+    const items = page?.items ?? [];
+    const end = items.findLast(i => i.turnEnd)?.turnEnd ?? null;
+    const last = items.findLast(i => i.role === 'assistant' && i.content?.text);
+    const turn = this.turn; this.turn = null;
+    turn.resolve({ lastText: last?.content?.text ?? '', turnEnd: end, error: p.settlement === 'error' ? p.reasonCode ?? 'error' : null, settlement: p.settlement });
+  }
+
+  // Questions and approvals waiting for the member, as the page shows them.
+  async #poll() {
+    if (!this.sessionRef || !this.turn) return;
+    const base = `/api/v1/sessions/${encodeURIComponent(this.sessionRef)}`;
+    const asked = (await this.http('GET', `${base}/questions`)).value?.questions ?? [];
+    for (const entry of asked) {
+      if (this.answered.has(entry.questionRef)) continue;
+      this.answered.add(entry.questionRef);
+      const answers = [];
+      for (const q of entry.questions) {
+        const a = await this.hooks.onQuestion(q);
+        answers.push(a.skipped ? null : a.other != null ? { selected: [], other: a.other } : { selected: a.selected, other: null });
+      }
+      const body = answers.every(a => a === null) ? { skipped: true } : { answers: answers.map(a => a ?? { selected: [0], other: null }) };
+      await this.http('POST', `${base}/questions/${encodeURIComponent(entry.questionRef)}/answer`, body);
+    }
+    const snapshot = (await this.http('GET', `${base}/inspection/snapshot`)).value;
+    for (const summary of snapshot?.approvals?.pending ?? []) {
+      const ref = summary.approvalRef; if (!ref || this.answered.has(ref)) continue;
+      this.answered.add(ref);
+      const req = (await this.http('GET', `${base}/inspection/approvals/${encodeURIComponent(ref)}`)).value;
+      const allow = await this.hooks.onApproval({ title: req?.action?.summary ?? req?.title ?? ref });
+      await this.http('POST', `/api/v1/approvals/${encodeURIComponent(ref)}/decision`, { schemaVersion: 1, version: 'piagent-webui-approval-v1', recordType: 'decision', approvalRef: ref,
+        decisionId: `decision.${randomUUID()}`, decisionToken: req.decisionToken, identity: req.identity, actionDigest: req.action?.actionDigest, expectedRevisions: req.expectedRevisions,
+        decision: allow ? 'allow' : 'deny', reason: null, decidedAt: new Date().toISOString(), expiresAt: req.expiresAt, decisionSurface: 'webui', executor: 'pi-guard', directExecution: false });
+    }
+  }
+
+  async send(text, delivery = 'new') {
+    const messageRequestId = opaque('message');
+    if (delivery !== 'new' && this.turn && this.operationRef) {
+      await this.#command('session.send', { delivery: delivery === 'steer' ? 'steer' : 'follow-up', message: text, messageRequestId, expectedOperationRef: this.operationRef, attachmentRefs: [] });
+      return;
+    }
+    let resolve; const ended = new Promise(r => { resolve = r; });
+    this.turn = { resolve, ended };
+    const receipt = this.sessionRef
+      ? await this.#command('session.send', { delivery: 'new-operation', message: text, messageRequestId, expectedOperationRef: null, attachmentRefs: [] })
+      : await this.#command('session.create', { ...this.create, message: text, messageRequestId });
+    if (!receipt?.sessionRef && !this.sessionRef) { const t = this.turn; this.turn = null; t.resolve({ lastText: '', turnEnd: null, error: receipt?.error?.code ?? receipt?.resultCode ?? 'create-failed' }); return; }
+    this.sessionRef ??= receipt.sessionRef;
+    if (receipt.operationRef) this.operationRef = receipt.operationRef;
+  }
+  nextTurnEnd() { return this.turn ? this.turn.ended : Promise.resolve({ lastText: '', turnEnd: null, error: 'no turn running' }); }
+  async stop() { if (this.operationRef) await this.#command('session.abort', { operationRef: this.operationRef, clearQueued: true }); }
+
+  // soft: the member reloads the page (or reopens it from Agent Watch): a
+  // new login and socket while the conversation keeps running in the
+  // Gateway. hard: the Gateway itself goes away mid-task (a Piagent update,
+  // the dashboard closed) and is started again; the running turn is gone and
+  // the member writes again in the same conversation.
+  async restart({ hard = false } = {}) {
+    const owner = hard ? this.gatewayOwner : null;
+    this.closing = true; this.ws?.close();
+    if (owner && owner.exitCode === null) {
+      const gone = new Promise(r => owner.once('exit', r));
+      owner.kill('SIGTERM'); await Promise.race([gone, wait(30_000)]);
+      this.gatewayOwner = null;
+      const turn = this.turn; this.turn = null; this.operationRef = null;
+      turn?.resolve({ lastText: '', turnEnd: null, error: null, restarted: true });
+    }
+    this.closing = false;
+    await this.#login(); this.ws = await this.#connect();
+    return owner ? 'reopened' : 'reattached';
+  }
+
+  // A Gateway this run started stops with it, as when its window closes.
+  async close() {
+    this.closing = true; clearInterval(this.poller); this.ws?.close();
+    const owner = this.gatewayOwner;
+    if (!owner || owner.exitCode !== null) return;
+    const gone = new Promise(r => owner.once('exit', r));
+    owner.kill('SIGTERM');
+    await Promise.race([gone, wait(30_000)]);
+  }
+}

package/evals/private-holdout-v1/public-exposure.v2.json

@@ -256,8 +256,8 @@
     },
     {
       "path": "tests",
-      "fileCount": 567,
-      "sha256": "bdfa32dcbc3679d30a684b3307c029946a25ca9db7743af48c41bce47b1d7b5a"
+      "fileCount": 570,
+      "sha256": "49831d3854e0342c89dc2e8dc28aaa0b9a0e6ae7870e1ab5c7ab003fee12e787"
     }
   ],
   "custodianComparisonRules": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@piagent/platform",
-  "version": "1.36.1",
+  "version": "1.37.1",
   "license": "MIT",
   "publishConfig": {
     "access": "public",
@@ -52,6 +52,7 @@
     "docs-site/",
     "!docs-site/assets/shots/",
     "evals/",
+    "!evals/long-task-soak-v1/fixtures/",
     "packs/",
     "packages/piagent-core/",
     "!packages/piagent-core/extensions/core-services.js",

package/packages/piagent-core/managed/helper-run.mjs

@@ -15,6 +15,7 @@ import { HELPER_CALLS, HELPER_SETUP, READ_TOOLS, countedCheck } from './helper-r
 import { trackToolFailures } from './tool-failures.mjs';
 import { helperMemoryTools, renewer } from './memory.mjs';
 import { answeredSettlement, failedSettlement, settleHelper, OUTCOME_WORDS } from './helper-settlement.mjs';
+import { viaLink } from './studio-link.mjs';
 
 const textContent = result => result.content?.filter(p => p.type === 'text').map(p => p.text).join('\n') ?? '';
 // A helper failure that another model of its role may not have: the model's
@@ -53,7 +54,16 @@ export async function runHelper(managed, role, task, signal, harness, { verifyGr
     }
   }
 }
+// A helper's role closes through the same wait as the run's close: a passing
+// failure of Agent Watch's way to Studio is asked again. A close that still
+// fails leaves the helper's answer standing: the role closes with the run,
+// and the next helper of the role closes it first.
+export async function closeRole(managed, role) {
+  try { await viaLink(managed.studioLink, () => managed.broker.request('close', { role })); managed.unclosed?.delete(role); }
+  catch { (managed.unclosed ??= new Set()).add(role); }
+}
 async function helperAttempt(managed, role, task, signal, harness, snapshot, parts, { verifyGrant, patiently, title }) {
+  if (managed.unclosed?.has(role)) await closeRole(managed, role);
   // A long local tool step may outlive the main lease. Refresh authority
   // before asking for a child; the broker serializes concurrent renewals,
   // and main's own searches running beside it share managed renew.
@@ -143,6 +153,6 @@ async function helperAttempt(managed, role, task, signal, harness, snapshot, par
   finally {
     for (const session of sessions) session.dispose(); await boundary?.dispose();
     await settleHelper(managed, renew.lease, report); // with the role's current token, before it closes
-    await managed.broker.request('close', { role });
+    await closeRole(managed, role);
   }
 }