@worca/app 1.5.0 → 1.6.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/docker/compose.broker.yml +79 -0
- package/docker/compose.isolation.yml +40 -0
- package/package.json +2 -1
- package/src/broker/config.mjs +200 -0
- package/src/broker/copilot.mjs +124 -0
- package/src/broker/limits.mjs +88 -0
- package/src/broker/main.mjs +129 -0
- package/src/broker/scrub.mjs +45 -0
- package/src/broker/service.mjs +558 -0
- package/src/broker/slots.mjs +180 -0
- package/src/broker/store.mjs +171 -0
- package/src/broker/tokens.mjs +116 -0
- package/src/broker/ui/page.css +54 -0
- package/src/broker/ui/page.html +25 -0
- package/src/broker/ui/page.mjs +202 -0
- package/src/broker/ui-server.mjs +217 -0
- package/src/broker/usage.mjs +108 -0
- package/src/broker/vault.mjs +37 -0
- package/src/cli/models.mjs +46 -14
- package/src/cli/render.mjs +21 -0
- package/src/cli/runs.mjs +296 -0
- package/src/cli/worca-cc.mjs +14 -2
- package/src/core/agent-pool.mjs +76 -0
- package/src/core/artifacts.mjs +40 -8
- package/src/core/ask/events.mjs +11 -0
- package/src/core/ask/html-text.mjs +113 -0
- package/src/core/ask/limits.mjs +5 -0
- package/src/core/ask/mcp-stdio.mjs +74 -20
- package/src/core/ask/prompt.mjs +25 -6
- package/src/core/ask/spawn.mjs +55 -5
- package/src/core/ask/store.mjs +1 -1
- package/src/core/ask/tools.mjs +66 -1
- package/src/core/ask/turn.mjs +52 -3
- package/src/core/ask/web-access.mjs +26 -0
- package/src/core/ask/web-deps.mjs +56 -0
- package/src/core/ask/web-fetch.mjs +271 -0
- package/src/core/ask/web-proposal.mjs +76 -0
- package/src/core/auto/classify.mjs +20 -8
- package/src/core/auto/runnable.mjs +28 -0
- package/src/core/billing.mjs +53 -0
- package/src/core/bridge/errors.mjs +77 -5
- package/src/core/bridge/openrouter.mjs +59 -0
- package/src/core/bridge/provider-ops.mjs +165 -9
- package/src/core/bridge/providers/endpoint.mjs +112 -7
- package/src/core/bridge/registry.mjs +13 -0
- package/src/core/bridge/server.mjs +16 -3
- package/src/core/bridge/telemetry.mjs +44 -12
- package/src/core/bridge/translate/common.mjs +31 -0
- package/src/core/bridge/translate/request.mjs +4 -1
- package/src/core/bridge/translate/response.mjs +3 -0
- package/src/core/bridge/translate/schema-keywords.mjs +75 -0
- package/src/core/bridge/translate/stream.mjs +50 -4
- package/src/core/bridge/upstream.mjs +102 -17
- package/src/core/broker-boot.mjs +57 -0
- package/src/core/broker-client.mjs +206 -0
- package/src/core/broker-guard.mjs +112 -0
- package/src/core/broker-routing.mjs +138 -0
- package/src/core/claude-auth.mjs +29 -0
- package/src/core/claude-runner.mjs +200 -9
- package/src/core/config.mjs +9 -12
- package/src/core/failure-policy.mjs +4 -2
- package/src/core/git-info.mjs +49 -5
- package/src/core/github-credentials.mjs +44 -1
- package/src/core/graph/script-runner.mjs +3 -1
- package/src/core/list-prices.mjs +29 -0
- package/src/core/mcp-secrets.mjs +80 -0
- package/src/core/metrics/sync.mjs +2 -1
- package/src/core/model-env.mjs +72 -0
- package/src/core/model-test.mjs +17 -5
- package/src/core/onboarding.mjs +12 -6
- package/src/core/openrouter-free.mjs +159 -0
- package/src/core/orchestrator.mjs +100 -12
- package/src/core/policy/effective.mjs +18 -1
- package/src/core/policy/local.mjs +4 -1
- package/src/core/policy/registry.mjs +12 -2
- package/src/core/preflight.mjs +116 -0
- package/src/core/recoverable-error.mjs +95 -0
- package/src/core/recovery-backoff.mjs +84 -0
- package/src/core/redact.mjs +25 -0
- package/src/core/run-context.mjs +6 -0
- package/src/core/run-harness.mjs +112 -30
- package/src/core/run-report.mjs +2 -1
- package/src/core/settings.mjs +90 -5
- package/src/core/title.mjs +7 -2
- package/src/core/web-allowlist.mjs +95 -0
- package/ui/public/app.js +244 -43
- package/ui/public/ask-model.mjs +1 -0
- package/ui/public/ask-panel.mjs +160 -23
- package/ui/public/bridge-view.mjs +175 -6
- package/ui/public/chat-settings-view.mjs +52 -1
- package/ui/public/credential-badges.mjs +63 -0
- package/ui/public/credentials-view.mjs +57 -0
- package/ui/public/index.html +33 -6
- package/ui/public/models-view.mjs +19 -1
- package/ui/public/openrouter-free-view.mjs +118 -0
- package/ui/public/stats-view.mjs +56 -0
- package/ui/public/style.css +99 -50
- package/ui/public/team-policy-view.mjs +2 -1
- package/ui/public/ws-seq.mjs +24 -0
- package/ui/server.mjs +319 -23
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
// accepted on read, downgraded to soft with a warning — a later version enforces it without a
|
|
16
16
|
// format change). Every field lists the kinds it accepts; the editor disables the rest.
|
|
17
17
|
|
|
18
|
+
import { domainError } from '../web-allowlist.mjs';
|
|
19
|
+
|
|
18
20
|
export const POLICY_SCHEMA = 1;
|
|
19
21
|
export const KINDS = Object.freeze(['default', 'soft', 'hard']);
|
|
20
22
|
export const ON_BREACH = Object.freeze(['pause', 'warn']);
|
|
@@ -41,6 +43,10 @@ export const FIELDS = Object.freeze([
|
|
|
41
43
|
{ key: 'cost.humanRateUsd', group: 'cost', label: 'Developer rate (USD/h)', help: 'Prices the estimated human hours behind "Saved". A developer\'s own rate wins when set.', type: 'usd', kinds: ['default'], local: 'humanRateUsdPerHour' },
|
|
42
44
|
{ key: 'ask.maxTurns', group: 'ask', label: 'Turn limit', help: 'Ask Worca agentic turns per chat turn.', type: 'int', min: 1, max: 500, kinds: ['default'], local: 'askMaxTurns' },
|
|
43
45
|
{ key: 'ask.maxBudgetUsd', group: 'ask', label: 'Per-turn cost cap (USD)', help: 'Ask Worca per-turn cap; null means no cap.', type: 'usd-or-null', min: 0.1, max: 100, kinds: ['default'], local: 'askMaxBudgetUsd' },
|
|
46
|
+
// The two web fields only ever NARROW, and bind (no "go past it"): web access is opt-in per developer, and the policy
|
|
47
|
+
// branch is writable by whoever can push to the repo — so a policy may switch it off or cap the hosts, never widen.
|
|
48
|
+
{ key: 'ask.webEnabled', group: 'ask', label: 'Web access', help: 'Off switches Ask Worca web access off for chats pinned to this project. A policy can never switch it on — each developer opts in.', type: 'bool', kinds: ['soft'], offOnly: true, local: 'askWeb.enabled' },
|
|
49
|
+
{ key: 'ask.webAllowedDomains', group: 'ask', label: 'Web allowlist', help: 'The most a developer may allow (example.com or *.example.com): hosts outside this list are dropped from their own Ask web allowlist. It never adds a host.', type: 'string[]', kinds: ['soft'], domains: true, local: 'askWeb.allowedDomains' },
|
|
44
50
|
{ key: 'guardrails.default', group: 'guardrails', label: 'Default set', help: 'What the New pipeline picker starts on. A built-in id, a user set id, or gp:<name> for a set this policy ships.', type: 'string', kinds: ['default'] },
|
|
45
51
|
{ key: 'guardrails.minimum', group: 'guardrails', label: 'Minimum tier', help: 'A run whose set ranks below this warns and is recorded.', type: 'enum', values: TIERS, kinds: ['soft'] },
|
|
46
52
|
{ key: 'models.allowed', group: 'models', label: 'Allowed models', help: 'A chosen step model outside this list warns and is recorded. Others still run.', type: 'string[]', kinds: ['soft'] },
|
|
@@ -98,13 +104,17 @@ export function validateValue(meta, value) {
|
|
|
98
104
|
if (value === null) return null;
|
|
99
105
|
return finiteNum(value) && value >= (meta.min ?? 0) && value <= (meta.max ?? Infinity) ? null : `must be null or a number between ${meta.min} and ${meta.max}`;
|
|
100
106
|
case 'int': return Number.isInteger(value) && value >= (meta.min ?? -Infinity) && value <= (meta.max ?? Infinity) ? null : `must be an integer between ${meta.min} and ${meta.max}`;
|
|
101
|
-
case 'bool':
|
|
107
|
+
case 'bool':
|
|
108
|
+
if (typeof value !== 'boolean') return 'must be true or false';
|
|
109
|
+
return meta.offOnly && value !== false ? 'a team policy can only switch web access off — turning it on is each developer\'s own choice' : null;
|
|
102
110
|
case 'enum': return meta.values.includes(value) ? null : `must be one of ${meta.values.join(' | ')}`;
|
|
103
111
|
case 'string': return typeof value === 'string' && value.trim() && value.length <= TEXT_MAX ? null : 'must be a non-empty string';
|
|
104
112
|
case 'semver': return typeof value === 'string' && SEMVER_RE.test(value) ? null : 'must be a version like 1.4.0';
|
|
105
113
|
case 'string[]':
|
|
106
114
|
if (!Array.isArray(value)) return 'must be a list';
|
|
107
|
-
|
|
115
|
+
if (!value.every((x) => typeof x === 'string' && x.trim() && x.length <= TEXT_MAX)) return 'every entry must be a non-empty string';
|
|
116
|
+
if (meta.domains) { for (const x of value) { const e = domainError(x); if (e) return e; } }
|
|
117
|
+
return null;
|
|
108
118
|
case 'steps': {
|
|
109
119
|
if (!isPlainObject(value)) return 'must be an object of role → { model, effort }';
|
|
110
120
|
for (const [role, sel] of Object.entries(value)) {
|
package/src/core/preflight.mjs
CHANGED
|
@@ -470,3 +470,119 @@ export async function probeClaudeCapabilities(bin = 'claude') {
|
|
|
470
470
|
const version = raw ? (/(\d+\.\d+\.\d+)/.exec(raw)?.[1] ?? raw.split(/\s+/)[0] ?? null) : null;
|
|
471
471
|
return { mcpConfig: !!help && help.includes('--mcp-config'), version };
|
|
472
472
|
}
|
|
473
|
+
|
|
474
|
+
// ── Is the Claude Code CLI signed in? ─────────────────────────────────────────
|
|
475
|
+
// Finding the binary is not enough: a signed-out CLI starts, then every agent
|
|
476
|
+
// exits 1 with "Not logged in" (a workspace scan got ~30 s into its first phase
|
|
477
|
+
// before failing). probeClaudeAuth asks `claude auth status` up front.
|
|
478
|
+
//
|
|
479
|
+
// Only 'signed-out' may ever block anything. 'unknown' — an older CLI without the
|
|
480
|
+
// `auth` command, a hang, unparseable output — never does, so a probe gap can
|
|
481
|
+
// only fall back to today's behaviour, never refuse a working setup.
|
|
482
|
+
|
|
483
|
+
/** Env vars that authenticate the CLI without a stored sign-in, which `claude
|
|
484
|
+
* auth status` does not (reliably) report: a custom endpoint or the model bridge
|
|
485
|
+
* (ANTHROPIC_BASE_URL + ANTHROPIC_AUTH_TOKEN), an API key or OAuth token, and
|
|
486
|
+
* the cloud transports (Bedrock / Vertex / Foundry). */
|
|
487
|
+
export const CLAUDE_AUTH_ENV_KEYS = Object.freeze([
|
|
488
|
+
'ANTHROPIC_BASE_URL', 'ANTHROPIC_AUTH_TOKEN', 'ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN',
|
|
489
|
+
]);
|
|
490
|
+
export const CLAUDE_AUTH_ENV_FLAGS = Object.freeze([
|
|
491
|
+
'CLAUDE_CODE_USE_BEDROCK', 'CLAUDE_CODE_USE_VERTEX', 'CLAUDE_CODE_USE_FOUNDRY',
|
|
492
|
+
]);
|
|
493
|
+
|
|
494
|
+
const flagOn = (v) => !!v && v !== '0' && String(v).toLowerCase() !== 'false';
|
|
495
|
+
|
|
496
|
+
/** The first env var that signs the CLI in on its own, or null. Pure. */
|
|
497
|
+
export function claudeAuthFromEnv(env = process.env) {
|
|
498
|
+
for (const k of CLAUDE_AUTH_ENV_KEYS) if (typeof env[k] === 'string' && env[k].trim()) return k;
|
|
499
|
+
for (const k of CLAUDE_AUTH_ENV_FLAGS) if (flagOn(env[k])) return k;
|
|
500
|
+
return null;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Read `claude auth status` output (JSON by default, `--text` on some builds).
|
|
505
|
+
* @param {string} text stdout + stderr
|
|
506
|
+
* @returns {'signed-in'|'signed-out'|'unknown'}
|
|
507
|
+
*/
|
|
508
|
+
export function parseClaudeAuthStatus(text) {
|
|
509
|
+
const s = String(text || '').trim();
|
|
510
|
+
if (!s) return 'unknown';
|
|
511
|
+
const json = /\{[\s\S]*\}/.exec(s);
|
|
512
|
+
if (json) {
|
|
513
|
+
try {
|
|
514
|
+
const o = JSON.parse(json[0]);
|
|
515
|
+
if (o && typeof o.loggedIn === 'boolean') return o.loggedIn ? 'signed-in' : 'signed-out';
|
|
516
|
+
} catch { /* fall through to the text forms */ }
|
|
517
|
+
}
|
|
518
|
+
if (/\bnot (logged|signed) in\b|\blogged out\b|\bsigned out\b/i.test(s)) return 'signed-out';
|
|
519
|
+
if (/\b(logged|signed) in\b/i.test(s)) return 'signed-in';
|
|
520
|
+
return 'unknown';
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/** Run a command with any exit code: resolves {code, out} (stdout+stderr), or
|
|
524
|
+
* null on a spawn failure / timeout. `auth status` exits non-zero when signed out. */
|
|
525
|
+
function execAnyExit(cmd, args, { timeout = 8000 } = {}) {
|
|
526
|
+
return new Promise((resolveP) => {
|
|
527
|
+
let child;
|
|
528
|
+
try { child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }); }
|
|
529
|
+
catch { resolveP(null); return; }
|
|
530
|
+
let out = '';
|
|
531
|
+
let settled = false;
|
|
532
|
+
const done = (val) => { if (settled) return; settled = true; clearTimeout(timer); resolveP(val); };
|
|
533
|
+
const timer = setTimeout(() => { try { child.kill('SIGKILL'); } catch { /* ignore */ } done(null); }, timeout);
|
|
534
|
+
child.stdout?.on('data', (d) => { out += d.toString(); });
|
|
535
|
+
child.stderr?.on('data', (d) => { out += d.toString(); });
|
|
536
|
+
child.on('error', () => done(null));
|
|
537
|
+
child.on('close', (code) => done({ code, out }));
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/** The refusal every surface shows for a signed-out CLI (the API's `code`, and its text). */
|
|
542
|
+
export const CLAUDE_SIGNED_OUT_CODE = 'claude-signed-out';
|
|
543
|
+
export const CLAUDE_SIGNED_OUT_MESSAGE = "Claude Code isn't signed in. Run `claude` in a terminal and type /login, then try again.";
|
|
544
|
+
|
|
545
|
+
/** Is `message` the CLI's own signed-out failure ("Not logged in · Please run
|
|
546
|
+
* /login")? For surfaces that only learn it after the spawn. Pure. */
|
|
547
|
+
export function isClaudeSignedOutError(message) {
|
|
548
|
+
return /\bnot logged in\b|please run \/login/i.test(String(message || ''));
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
export const CLAUDE_AUTH_TTL_MS = 60_000;
|
|
552
|
+
const _authCache = new Map(); // exe -> { at, promise }
|
|
553
|
+
|
|
554
|
+
/** Test seam: forget remembered answers. */
|
|
555
|
+
export function clearClaudeAuthCache() { _authCache.clear(); }
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* Is the Claude Code CLI signed in? Never throws.
|
|
559
|
+
* - mock mode: never starts claude → {state:'unknown', source:'mock'}
|
|
560
|
+
* - an auth env var (see CLAUDE_AUTH_ENV_KEYS / _FLAGS) → {state:'signed-in', source:'env'}
|
|
561
|
+
* - otherwise `claude auth status`, remembered per binary for CLAUDE_AUTH_TTL_MS
|
|
562
|
+
* (concurrent callers share one spawn); `force` skips the remembered answer.
|
|
563
|
+
* @param {{bin?:string, env?:Record<string,string|undefined>, mock?:boolean, force?:boolean,
|
|
564
|
+
* now?:()=>number, run?:(exe:string, args:string[])=>Promise<{code:number,out:string}|null>}} [o]
|
|
565
|
+
* @returns {Promise<{state:'signed-in'|'signed-out'|'unknown', source:'mock'|'env'|'cli', detail:string|null}>}
|
|
566
|
+
*/
|
|
567
|
+
export async function probeClaudeAuth({
|
|
568
|
+
bin = 'claude', env = process.env, mock, force = false, now = Date.now, run = execAnyExit,
|
|
569
|
+
} = {}) {
|
|
570
|
+
const isMock = mock ?? (flagOn(env.WORCA_MOCK ?? env.ORCH_MOCK));
|
|
571
|
+
if (isMock) return { state: 'unknown', source: 'mock', detail: null };
|
|
572
|
+
// Credential broker (broker-client.mjs): the CLI is signed in per spawn with a
|
|
573
|
+
// broker token, never by worca's env or a stored login. Whether the PERSON has a
|
|
574
|
+
// key is the broker's per-slot status, not this probe's question.
|
|
575
|
+
if (typeof env.WORCA_BROKER_URL === 'string' && env.WORCA_BROKER_URL.trim()) return { state: 'signed-in', source: 'broker', detail: 'broker' };
|
|
576
|
+
const envKey = claudeAuthFromEnv(env);
|
|
577
|
+
if (envKey) return { state: 'signed-in', source: 'env', detail: envKey };
|
|
578
|
+
const name = bin && String(bin).trim() ? String(bin).trim() : 'claude';
|
|
579
|
+
const exe = resolveClaudeBin(name).bin;
|
|
580
|
+
const hit = _authCache.get(exe);
|
|
581
|
+
if (!force && hit && now() - hit.at < CLAUDE_AUTH_TTL_MS) return hit.promise;
|
|
582
|
+
const promise = Promise.resolve()
|
|
583
|
+
.then(() => run(exe, ['auth', 'status']))
|
|
584
|
+
.then((r) => ({ state: r ? parseClaudeAuthStatus(r.out) : 'unknown', source: 'cli', detail: null }))
|
|
585
|
+
.catch(() => ({ state: 'unknown', source: 'cli', detail: null }));
|
|
586
|
+
_authCache.set(exe, { at: now(), promise });
|
|
587
|
+
return promise;
|
|
588
|
+
}
|
|
@@ -24,6 +24,13 @@ export function classifyError(err) {
|
|
|
24
24
|
// stamp — including an explicit null — is authoritative.
|
|
25
25
|
if (err && typeof err === 'object' && err.errorClass !== undefined) return err.errorClass;
|
|
26
26
|
const msg = String((err && err.message) || err || '');
|
|
27
|
+
// The credential broker's refusals (a missing or rejected key, a dead token, nobody to
|
|
28
|
+
// bill) arrive as a 403 the CLI prints as "Failed to authenticate. API Error: 403
|
|
29
|
+
// worca-broker: …" — no 401, no authentication_error left in the text. Same class.
|
|
30
|
+
if (BROKER_AUTH_RE.test(msg)) return 'auth';
|
|
31
|
+
// OpenRouter's daily allowance of `:free` requests is spent: only its daily reset clears
|
|
32
|
+
// it, so it pauses like the session limit below instead of retrying as a 429.
|
|
33
|
+
if (FREE_DAILY_RE.test(msg)) return 'usage_limit';
|
|
27
34
|
if (/\b401\b|invalid authentication|authentication_error|please run .*login|not logged in/i.test(msg)) return 'auth';
|
|
28
35
|
// Session/usage caps that only clear after a multi-hour reset (the CLI prints
|
|
29
36
|
// "You've hit your session limit · resets 6pm"). Distinct from rate_limit (a
|
|
@@ -37,6 +44,94 @@ export function classifyError(err) {
|
|
|
37
44
|
return null;
|
|
38
45
|
}
|
|
39
46
|
|
|
47
|
+
// OpenRouter's `:free` models run on a donated provider pool that every OpenRouter
|
|
48
|
+
// user shares: its 429 ("temporarily rate-limited upstream", limit_source
|
|
49
|
+
// upstream_provider_shared_pool) arrives for a single request, whatever worca's
|
|
50
|
+
// max-concurrent setting. Still class rate_limit (a retry can clear it); only the
|
|
51
|
+
// final pause message changes, so it names the real cause and the real fixes.
|
|
52
|
+
const SHARED_POOL_RE = /rate-limited upstream|upstream_provider_shared_pool/i;
|
|
53
|
+
|
|
54
|
+
/** Whether a rate-limit error came from a provider's shared (free) pool. */
|
|
55
|
+
export function isSharedPoolRateLimit(err) {
|
|
56
|
+
const msg = String((err && typeof err === 'object' ? err.message : err) ?? '');
|
|
57
|
+
return SHARED_POOL_RE.test(msg);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The fix text a rate-limit pause carries ('' when there is nothing specific to say). */
|
|
61
|
+
export function rateLimitHint(err) {
|
|
62
|
+
if (!isSharedPoolRateLimit(err)) return '';
|
|
63
|
+
return "the provider's shared free pool is saturated — every user of this free model shares it, " +
|
|
64
|
+
"so this is not worca's max-concurrent setting. Use the paid variant, add your own provider key " +
|
|
65
|
+
'(BYOK) on the provider, or give the model a fallback model list';
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// OpenRouter's daily allowance for `:free` models (1000 requests a day on an account that
|
|
69
|
+
// bought $10 of credit, 50 below): its 429 names the limit ("Rate limit exceeded:
|
|
70
|
+
// free-models-per-day-high-balance", limit_source openrouter_free_tier_daily) and clears
|
|
71
|
+
// only at the daily reset, 00:00 UTC. The bridge adds "resets <ISO>" when OpenRouter says.
|
|
72
|
+
const FREE_DAILY_RE = /openrouter_free_tier_daily|free-models-per-day/i;
|
|
73
|
+
const FREE_DAILY_RESET_RE = /resets (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)/;
|
|
74
|
+
|
|
75
|
+
/** Whether an error is OpenRouter's spent daily allowance of free-model requests. */
|
|
76
|
+
export function isFreeDailyLimit(err) {
|
|
77
|
+
const msg = String((err && typeof err === 'object' ? err.message : err) ?? '');
|
|
78
|
+
return FREE_DAILY_RE.test(msg);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** When the free-model allowance comes back (ms): the reset the error names, else the next 00:00 UTC. */
|
|
82
|
+
export function freeDailyResetAt(err, now = Date.now()) {
|
|
83
|
+
const msg = String((err && typeof err === 'object' ? err.message : err) ?? '');
|
|
84
|
+
const m = FREE_DAILY_RESET_RE.exec(msg);
|
|
85
|
+
const named = m ? Date.parse(m[1]) : NaN;
|
|
86
|
+
if (Number.isFinite(named) && named > now) return named;
|
|
87
|
+
const d = new Date(now);
|
|
88
|
+
return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate() + 1);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** "3h 12m" / "12m" / "under a minute". Pure. */
|
|
92
|
+
export function untilText(ms) {
|
|
93
|
+
const min = Math.floor(Math.max(0, ms) / 60_000);
|
|
94
|
+
if (min < 1) return 'under a minute';
|
|
95
|
+
const h = Math.floor(min / 60);
|
|
96
|
+
return h ? `${h}h ${min % 60}m` : `${min}m`;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The pause text for a spent free-model allowance ('' for any other error): what ran out,
|
|
101
|
+
* when it comes back, and the two ways on. `used`/`limit` when a key reading has them.
|
|
102
|
+
*/
|
|
103
|
+
export function freeDailyHint(err, { now = Date.now(), used = null, limit = null } = {}) {
|
|
104
|
+
if (!isFreeDailyLimit(err)) return '';
|
|
105
|
+
const at = freeDailyResetAt(err, now);
|
|
106
|
+
const hhmm = new Date(at).toISOString().slice(11, 16);
|
|
107
|
+
const count = Number.isFinite(limit) && limit > 0 ? ` (${Number.isFinite(used) ? used : limit} / ${limit})` : '';
|
|
108
|
+
return `OpenRouter's free-model requests for today are used up${count} — they reset at ${hhmm} UTC, in ${untilText(at - now)}. ` +
|
|
109
|
+
'Resume after the reset, or switch this step to a paid model';
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// The credential broker (src/broker/) answers in the provider's own error envelope
|
|
113
|
+
// with a `worca-broker:` message, so its refusals already land in the right class:
|
|
114
|
+
// a missing/rejected key or a dead token is `auth` (401), a spent budget is `quota`
|
|
115
|
+
// (its message says "quota reached"), a busy slot is `rate_limit` (429). Only the
|
|
116
|
+
// pause text differs: it says what to do, which is never "sign in to Claude Code".
|
|
117
|
+
const BROKER_RE = /worca-broker:/i;
|
|
118
|
+
/** The broker's credential refusals (classifyError reads them as `auth`). */
|
|
119
|
+
const BROKER_AUTH_RE = /worca-broker: (?:no .+ for \S+|your .+ was rejected|.+ key for \S+: |token expired or revoked|no token|this action has no signed-in person|cannot get a token)/i;
|
|
120
|
+
|
|
121
|
+
/** Whether an error came from the credential broker. */
|
|
122
|
+
export function isBrokerError(err) {
|
|
123
|
+
const msg = String((err && typeof err === 'object' ? err.message : err) ?? '');
|
|
124
|
+
return BROKER_RE.test(msg);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The fix text a pause caused by the broker carries ('' for any other error). */
|
|
128
|
+
export function brokerHint(err, cls = classifyError(err)) {
|
|
129
|
+
if (!isBrokerError(err)) return '';
|
|
130
|
+
if (cls === 'auth') return 'the credential broker refused this spawn: add or replace the key on the key page (Settings › My credentials links to it), then resume';
|
|
131
|
+
if (cls === 'quota') return 'a spending cap on the credential broker was reached: raise it on the key page or wait for the reset, then resume';
|
|
132
|
+
return '';
|
|
133
|
+
}
|
|
134
|
+
|
|
40
135
|
// Precedence for folding per-line classes into the one whole-text class — the
|
|
41
136
|
// SAME order as the regex chain above. First-match-wins there equals
|
|
42
137
|
// strongest-class-wins here, because every per-line match (the patterns are
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
// src/core/recovery-backoff.mjs
|
|
2
|
+
// The ONE backoff rule for recoverable errors (recoverable-error.mjs classes). The
|
|
3
|
+
// node retry loop (run-harness#_backoff) and worca's own helper calls — the Auto
|
|
4
|
+
// classifier, title generation — share it, so "how long does worca wait on a 429"
|
|
5
|
+
// has one answer.
|
|
6
|
+
//
|
|
7
|
+
// A rate limit waits longer than a network blip: a provider's 429 (a shared free
|
|
8
|
+
// pool, a per-minute cap) takes seconds to clear, and the CLI has already retried
|
|
9
|
+
// the request itself before exiting. Each wait honours a retry-after hint when the
|
|
10
|
+
// message carries one, and is capped so three retries stay within ~2 minutes.
|
|
11
|
+
// WORCA_RECOVERY_BACKOFF_MS overrides the base for every class (tests pin it to 0).
|
|
12
|
+
import { classifyError } from './recoverable-error.mjs';
|
|
13
|
+
|
|
14
|
+
export const RATE_LIMIT_BACKOFF_BASE_MS = 5_000;
|
|
15
|
+
export const NETWORK_BACKOFF_BASE_MS = 1_000;
|
|
16
|
+
/** Per-wait cap: 3 capped waits stay within ~2 minutes. */
|
|
17
|
+
export const MAX_RECOVERY_WAIT_MS = 40_000;
|
|
18
|
+
/** Retries a helper call gets (withRecoveryRetry) — the node loop's own budget is RECOVERY_MAX_AUTO_ATTEMPTS. */
|
|
19
|
+
export const HELPER_RETRY_ATTEMPTS = 3;
|
|
20
|
+
/** The classes a wait can clear. auth/quota/usage_limit never clear by waiting. */
|
|
21
|
+
export const RETRYABLE_CLASSES = Object.freeze(['rate_limit', 'network']);
|
|
22
|
+
|
|
23
|
+
/** The first wait for a class, before doubling. */
|
|
24
|
+
export function backoffBaseMs(cls, env = process.env) {
|
|
25
|
+
const n = Number(env?.WORCA_RECOVERY_BACKOFF_MS);
|
|
26
|
+
if (env?.WORCA_RECOVERY_BACKOFF_MS != null && env.WORCA_RECOVERY_BACKOFF_MS !== '' && Number.isFinite(n) && n >= 0) return n;
|
|
27
|
+
return cls === 'rate_limit' ? RATE_LIMIT_BACKOFF_BASE_MS : NETWORK_BACKOFF_BASE_MS;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** A retry-after hint in an error message ("retry-after: 12", "retry after 3 seconds", "1500ms"), in ms; null when absent. */
|
|
31
|
+
export function retryAfterMs(err) {
|
|
32
|
+
const msg = String((err && typeof err === 'object' ? err.message : err) ?? '');
|
|
33
|
+
const m = /retry[- ]after[:=\s]*(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|secs?|seconds?)?/i.exec(msg);
|
|
34
|
+
if (!m) return null;
|
|
35
|
+
const n = Number(m[1]);
|
|
36
|
+
if (!Number.isFinite(n) || n < 0) return null;
|
|
37
|
+
return /^m/i.test(m[2] || '') ? Math.round(n) : Math.round(n * 1000);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The wait before retry `attempt` (1-based): base·2^(attempt-1), at least a retry-after hint, at most MAX_RECOVERY_WAIT_MS. */
|
|
41
|
+
export function recoveryDelayMs({ cls, attempt = 1, err = null, env = process.env } = {}) {
|
|
42
|
+
const base = backoffBaseMs(cls, env);
|
|
43
|
+
if (!base) return 0;
|
|
44
|
+
const exp = base * Math.pow(2, Math.max(0, attempt - 1));
|
|
45
|
+
const hint = retryAfterMs(err) ?? 0;
|
|
46
|
+
return Math.min(MAX_RECOVERY_WAIT_MS, Math.max(exp, hint));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Wait `ms`, resolving early (never rejecting) when `signal` aborts. */
|
|
50
|
+
export function sleepAbortable(ms, signal) {
|
|
51
|
+
if (!ms || ms <= 0 || signal?.aborted) return Promise.resolve();
|
|
52
|
+
return new Promise((res) => {
|
|
53
|
+
// Not unref'd: a helper call waiting out a 429 is real pending work — an
|
|
54
|
+
// unref'd timer would let a headless CLI process exit mid-wait.
|
|
55
|
+
const t = setTimeout(done, ms);
|
|
56
|
+
function done() { clearTimeout(t); signal?.removeEventListener?.('abort', done); res(); }
|
|
57
|
+
signal?.addEventListener?.('abort', done, { once: true });
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Run `fn`, retrying it on a retryable class (rate_limit / network) with the
|
|
63
|
+
* shared backoff. Anything else — and the last error once `attempts` retries are
|
|
64
|
+
* spent, or once `signal` aborts — is rethrown unchanged.
|
|
65
|
+
* @param {() => Promise<any>} fn
|
|
66
|
+
* `classes` narrows what is retried (title generation retries only rate_limit: an
|
|
67
|
+
* unspawnable CLI is stamped network, and a cosmetic call should not wait it out).
|
|
68
|
+
* @param {{attempts?:number, classes?:string[], signal?:AbortSignal, env?:object,
|
|
69
|
+
* onRetry?:(w:{attempt:number, cls:string, delayMs:number, err:Error}) => void}} [o]
|
|
70
|
+
*/
|
|
71
|
+
export async function withRecoveryRetry(fn, { attempts = HELPER_RETRY_ATTEMPTS, classes = RETRYABLE_CLASSES, signal, env = process.env, onRetry } = {}) {
|
|
72
|
+
for (let attempt = 1; ; attempt++) {
|
|
73
|
+
try {
|
|
74
|
+
return await fn();
|
|
75
|
+
} catch (err) {
|
|
76
|
+
const cls = classifyError(err);
|
|
77
|
+
if (attempt > attempts || !classes.includes(cls) || signal?.aborted || err?.name === 'AbortError') throw err;
|
|
78
|
+
const delayMs = recoveryDelayMs({ cls, attempt, err, env });
|
|
79
|
+
try { onRetry?.({ attempt, cls, delayMs, err }); } catch { /* a logger must not break the retry */ }
|
|
80
|
+
await sleepAbortable(delayMs, signal);
|
|
81
|
+
if (signal?.aborted) throw err;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// src/core/redact.mjs
|
|
2
|
+
// Broker spawn tokens in agent output (plans/credential-broker-design.html §6.10).
|
|
3
|
+
// A token only works on the broker's private port and only while its spawn lives,
|
|
4
|
+
// but an agent can still print its own env into a transcript every teammate sees.
|
|
5
|
+
// The `wbt_` prefix makes it recognisable; this removes it wherever worca stores or
|
|
6
|
+
// shows agent output. Pure, cheap on text without the prefix.
|
|
7
|
+
|
|
8
|
+
export const BROKER_TOKEN_RE = /\bwbt_[A-Za-z0-9_-]{20,}/g;
|
|
9
|
+
export const REDACTED_TOKEN = 'wbt_[redacted]';
|
|
10
|
+
|
|
11
|
+
/** `text` with every broker token replaced. Non-strings come back unchanged. */
|
|
12
|
+
export function redactSecrets(text) {
|
|
13
|
+
if (typeof text !== 'string' || !text.includes('wbt_')) return text;
|
|
14
|
+
return text.replace(BROKER_TOKEN_RE, REDACTED_TOKEN);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** Deep copy of a JSON-like value with every string redacted (events, tool results). */
|
|
18
|
+
export function redactDeep(value, depth = 0) {
|
|
19
|
+
if (typeof value === 'string') return redactSecrets(value);
|
|
20
|
+
if (depth > 32 || value === null || typeof value !== 'object') return value;
|
|
21
|
+
if (Array.isArray(value)) return value.map((v) => redactDeep(v, depth + 1));
|
|
22
|
+
const out = {};
|
|
23
|
+
for (const [k, v] of Object.entries(value)) out[k] = redactDeep(v, depth + 1);
|
|
24
|
+
return out;
|
|
25
|
+
}
|
package/src/core/run-context.mjs
CHANGED
|
@@ -41,6 +41,7 @@ import { contextMaxBytesPerFile, contextMaxBytesTotal, skillMount, defaultRoot }
|
|
|
41
41
|
import { readRunManifest, updateRunManifest } from './run-manifest.mjs';
|
|
42
42
|
import { isValidSkillName } from './skills.mjs';
|
|
43
43
|
import { mergePermissionRules } from './guardrails.mjs';
|
|
44
|
+
import { screenMcpSecrets, mcpSecretsMode } from './mcp-secrets.mjs';
|
|
44
45
|
|
|
45
46
|
/**
|
|
46
47
|
* The `--allowedTools` grant shape this build emits for merged MCP servers.
|
|
@@ -1138,6 +1139,11 @@ export async function assembleRunContext({
|
|
|
1138
1139
|
members: liveMembers, projectsRoot: rootUsable ? projectsRoot : null, homeDir, isWorkspace, platform,
|
|
1139
1140
|
});
|
|
1140
1141
|
for (const w of mcp.warnings) warnings.push(w);
|
|
1142
|
+
// Secrets in these definitions reach the run's agents (mcp-secrets.mjs): with the
|
|
1143
|
+
// credential broker on they are left out by default, otherwise named.
|
|
1144
|
+
const screened = screenMcpSecrets(mcp.servers, { mode: mcpSecretsMode() });
|
|
1145
|
+
for (const w of screened.warnings) warnings.push(w);
|
|
1146
|
+
mcp.servers = screened.servers;
|
|
1141
1147
|
const written = Object.keys(mcp.servers).sort();
|
|
1142
1148
|
let mcpConfigPath = null;
|
|
1143
1149
|
if (written.length) {
|
package/src/core/run-harness.mjs
CHANGED
|
@@ -55,8 +55,8 @@ import {
|
|
|
55
55
|
probeClaudeCapabilities, explainUnspawnableClaude,
|
|
56
56
|
} from './preflight.mjs';
|
|
57
57
|
import { fanoutCap, mapWithCap } from './fanout.mjs';
|
|
58
|
-
import { resolveStepModels, observeModelCost, resolveModelCost, modelCostConfig, readTeamMetricsPrefs } from './config.mjs';
|
|
59
|
-
import { bridgeCallsFor, forgetBridgeTag } from './bridge/telemetry.mjs';
|
|
58
|
+
import { resolveStepModels, observeModelCost, resolveModelCost, modelCostConfig, readTeamMetricsPrefs, catalogHasModel } from './config.mjs';
|
|
59
|
+
import { bridgeCallsFor, bridgeCostFor, forgetBridgeTag } from './bridge/telemetry.mjs';
|
|
60
60
|
import { readGuardrailSet } from './guardrail-store.mjs';
|
|
61
61
|
import { unionGuardrails, guardrailsToPermissionRules, mergePermissionRules } from './guardrails.mjs';
|
|
62
62
|
import { collectRequiredSkills, validateSkills, injectSkills, pluginSkillDirs } from './skills.mjs';
|
|
@@ -66,10 +66,16 @@ import {
|
|
|
66
66
|
isValidSourceRef, snapshotWorktreePatch,
|
|
67
67
|
} from './worktree.mjs';
|
|
68
68
|
import { readPluginsLock, pluginCurrentDir } from './plugins-lock.mjs'; // §9.4 disabled-plugin hint
|
|
69
|
-
import { classifyError } from './recoverable-error.mjs';
|
|
69
|
+
import { classifyError, rateLimitHint, brokerHint, freeDailyHint } from './recoverable-error.mjs';
|
|
70
|
+
import { cachedFreeDailyCounts } from './openrouter-free.mjs';
|
|
71
|
+
import { withBillTo, currentBillTo } from './billing.mjs';
|
|
72
|
+
import { brokerEnabled, brokerInfo, personSlots } from './broker-client.mjs';
|
|
73
|
+
import { mockEnabled } from './claude-runner.mjs';
|
|
74
|
+
import { modelSlot, manifestModels, missingCredentials, describeMissing } from './broker-routing.mjs';
|
|
75
|
+
import { recoveryDelayMs, sleepAbortable } from './recovery-backoff.mjs';
|
|
70
76
|
import {
|
|
71
77
|
resolveFailure, isTerminal, markTerminal, answerFromDecision,
|
|
72
|
-
REASON, pauseConsequences, describePauseReason,
|
|
78
|
+
REASON, pauseConsequences, describePauseReason, RECOVERY_MAX_AUTO_ATTEMPTS,
|
|
73
79
|
} from './failure-policy.mjs';
|
|
74
80
|
import { recordRunMetrics } from './metrics/record.mjs';
|
|
75
81
|
// Team policy (team-policy design §6–§7): the document a run's cost gates fold in, its
|
|
@@ -626,8 +632,27 @@ export class RunHarness extends EventEmitter {
|
|
|
626
632
|
bin: this.opts.claude?.bin,
|
|
627
633
|
permissionMode: this.opts.claude?.permissionMode || 'acceptEdits',
|
|
628
634
|
model: this.opts.claude?.model,
|
|
635
|
+
effort: this.opts.claude?.effort,
|
|
629
636
|
mock: !!this.opts.claude?.mock,
|
|
630
637
|
};
|
|
638
|
+
// A resumed run keeps the model it was started with (`worca --model`, the UI's
|
|
639
|
+
// start pair): the resume sites pass none, so it rides the resume point — which
|
|
640
|
+
// _buildResumePoint rewrites at every pause from this.claude — like memoryScope
|
|
641
|
+
// below. A resume that names its own model wins. A saved model that left the
|
|
642
|
+
// catalog is dropped (the run falls back to the default, as before) and
|
|
643
|
+
// resume() says so once the run log is bound.
|
|
644
|
+
this._staleResumeModel = null;
|
|
645
|
+
{
|
|
646
|
+
const saved = this.opts.resume?.resumePoint?.claude;
|
|
647
|
+
if (saved && !this.claude.model && typeof saved.model === 'string' && saved.model) {
|
|
648
|
+
if (catalogHasModel(saved.model)) {
|
|
649
|
+
this.claude.model = saved.model;
|
|
650
|
+
if (!this.claude.effort && typeof saved.effort === 'string' && saved.effort) this.claude.effort = saved.effort;
|
|
651
|
+
} else {
|
|
652
|
+
this._staleResumeModel = saved.model;
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
}
|
|
631
656
|
// The mock runner routes EVERY dontAsk spawn to the Ask Worca mock (claude-runner.mjs
|
|
632
657
|
// runMock, rule R-F), so a mock pipeline role under dontAsk writes no artifact and
|
|
633
658
|
// the run dies at its first artifact read with no hint why. Fail at construction
|
|
@@ -967,8 +992,16 @@ export class RunHarness extends EventEmitter {
|
|
|
967
992
|
const where = label || nc?.key || ctx?.nodeId || 'orchestrator';
|
|
968
993
|
const meta = ctx ? { nodeId: ctx.nodeId, executionId: ctx.executionId, cycle: ctx.ordinal } : {};
|
|
969
994
|
const line = firstLine(err?.message || (err == null ? '' : String(err))) || 'unknown error';
|
|
995
|
+
// A shared-pool 429 (OpenRouter `:free`) names its real cause and fixes —
|
|
996
|
+
// otherwise "rate limited" reads as worca's own max-concurrent setting.
|
|
997
|
+
// A credential-broker refusal (a missing key, a spent cap) says where to fix it.
|
|
998
|
+
const hint = reason !== REASON.RECOVERABLE ? ''
|
|
999
|
+
: cls === 'rate_limit' ? rateLimitHint(err) : brokerHint(err, cls);
|
|
1000
|
+
// OpenRouter's spent daily free requests: say what ran out and when it comes back,
|
|
1001
|
+
// not the raw 429 line (freeDailyHint is '' for any other usage limit).
|
|
1002
|
+
const freeDaily = reason === REASON.USAGE_LIMIT ? freeDailyHint(err, cachedFreeDailyCounts(currentBillTo())) : '';
|
|
970
1003
|
const text = detail ?? (reason === REASON.ERROR ? errorDetail(err)
|
|
971
|
-
: reason === REASON.RECOVERABLE ? `${cls || 'recoverable'}: ${line}` : line);
|
|
1004
|
+
: reason === REASON.RECOVERABLE ? `${cls || 'recoverable'}: ${line}${hint ? ` — ${hint}` : ''}` : (freeDaily || line));
|
|
972
1005
|
if (reason === REASON.ERROR) {
|
|
973
1006
|
// The ONE error-level line, written BEFORE the pause sentinel the caller
|
|
974
1007
|
// throws next (a pause/abort is never logged as a failure).
|
|
@@ -986,9 +1019,9 @@ export class RunHarness extends EventEmitter {
|
|
|
986
1019
|
this._log(where, 'warn', `${describePauseReason(reason)} — pausing for manual resume: ${text}`, meta);
|
|
987
1020
|
audit = `Pipeline **paused**: session/usage limit on ${where} — ${text}. Resume after the reset.`;
|
|
988
1021
|
} else if (reason === REASON.RECOVERABLE) {
|
|
989
|
-
this._log(where, 'warn', `recoverable ${cls || 'error'} error — pausing for manual resume: ${line}`,
|
|
1022
|
+
this._log(where, 'warn', `recoverable ${cls || 'error'} error — pausing for manual resume: ${line}${hint ? ` — ${hint}` : ''}`,
|
|
990
1023
|
{ ...meta, ...(err?.stream ? { stream: err.stream } : {}) });
|
|
991
|
-
audit = `Pipeline **paused**: recoverable ${cls || 'error'} error on ${where} — ${line}. Resume to retry.`;
|
|
1024
|
+
audit = `Pipeline **paused**: recoverable ${cls || 'error'} error on ${where} — ${line}.${hint ? ` ${hint[0].toUpperCase()}${hint.slice(1)}.` : ''} Resume to retry.`;
|
|
992
1025
|
} else {
|
|
993
1026
|
this._log(where, 'warn', `${text} — pausing for manual resume`, meta);
|
|
994
1027
|
audit = `Pipeline **paused**: ${text}.`;
|
|
@@ -1001,8 +1034,15 @@ export class RunHarness extends EventEmitter {
|
|
|
1001
1034
|
/**
|
|
1002
1035
|
* Execute the full pipeline. Resolves with { status, pipelineDir } on success
|
|
1003
1036
|
* or stop; rejects only on unexpected internal errors (it emits 'error' too).
|
|
1037
|
+
*
|
|
1038
|
+
* Every spawn of the run is billed to the person who started it (billing.mjs,
|
|
1039
|
+
* credential broker): the whole loop runs inside that async context.
|
|
1004
1040
|
*/
|
|
1005
|
-
|
|
1041
|
+
run() {
|
|
1042
|
+
return withBillTo(this.opts.startedBy || currentBillTo(), () => this._run());
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
async _run() {
|
|
1006
1046
|
try {
|
|
1007
1047
|
this.state.startedAt = new Date().toISOString();
|
|
1008
1048
|
this._setStatus('running');
|
|
@@ -1041,6 +1081,9 @@ export class RunHarness extends EventEmitter {
|
|
|
1041
1081
|
this.toolInstruction = tools.instruction || '';
|
|
1042
1082
|
this.state.tools = tools;
|
|
1043
1083
|
this.stepModels = stepModels;
|
|
1084
|
+
// Credential broker: every model this run will spawn needs its person's key; refuse
|
|
1085
|
+
// NOW, naming what's missing, instead of pausing mid-run at the first node that needs it.
|
|
1086
|
+
await this._brokerPreflight(topology.manifest, stepModels);
|
|
1044
1087
|
await this._resolveGuardrails();
|
|
1045
1088
|
await this._resolvePolicy();
|
|
1046
1089
|
this._log(
|
|
@@ -1373,8 +1416,18 @@ export class RunHarness extends EventEmitter {
|
|
|
1373
1416
|
* artifacts exist from the original run, unless the point is stamped
|
|
1374
1417
|
* `setupIncomplete` (D7 replay), which re-runs whatever setup never finished.
|
|
1375
1418
|
* Resolves like run().
|
|
1419
|
+
*
|
|
1420
|
+
* Billed to whoever resumed it (the request's person, billing.mjs); a resume with
|
|
1421
|
+
* no person behind it (a restart's auto-resume) stays with the run's starter.
|
|
1376
1422
|
*/
|
|
1377
|
-
|
|
1423
|
+
resume() {
|
|
1424
|
+
const who = currentBillTo();
|
|
1425
|
+
const starter = this.resumeOpts?.row?.started_by ?? this.opts.startedBy ?? null;
|
|
1426
|
+
// Pays: whoever resumed. Runs as: the starter's agent user, whose HOME holds the sessions.
|
|
1427
|
+
return withBillTo(who && who !== 'local' ? who : (starter || who), () => this._resume(), { owner: starter || who });
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
async _resume() {
|
|
1378
1431
|
const saved = this.resumeOpts;
|
|
1379
1432
|
if (!saved?.row || !saved?.resumePoint) throw new Error('resume(): no saved pipeline provided');
|
|
1380
1433
|
const { row, resumePoint: rp, steps } = saved;
|
|
@@ -1428,6 +1481,9 @@ export class RunHarness extends EventEmitter {
|
|
|
1428
1481
|
this.state.pipelineDir = rp.pipelineDir;
|
|
1429
1482
|
this.logWriter.bind(rp.pipelineDir);
|
|
1430
1483
|
recordArtifact(row.id, RUN_LOG_KIND, RUN_LOG_FILE);
|
|
1484
|
+
if (this._staleResumeModel) {
|
|
1485
|
+
this._log('orchestrator', 'warn', `model ${JSON.stringify(this._staleResumeModel)} the run was started with is no longer in the catalog — resuming on the default model`);
|
|
1486
|
+
}
|
|
1431
1487
|
this.stepModels = rp.stepModels || null;
|
|
1432
1488
|
this.workflowId = rp.workflowId || this.workflowId;
|
|
1433
1489
|
// Pauses are counted only in _completePaused, so a crash-resume of an `interrupted`
|
|
@@ -3023,6 +3079,33 @@ export class RunHarness extends EventEmitter {
|
|
|
3023
3079
|
* recoverable-error gate surfaces it cleanly.
|
|
3024
3080
|
* @param {Iterable<string>} agentKeys the run's distinct agent keys, in launch order
|
|
3025
3081
|
*/
|
|
3082
|
+
/**
|
|
3083
|
+
* Credential-broker preflight (docs/credential-broker.md): the models in the manifest
|
|
3084
|
+
* (plus the step defaults and the run's own model, which an empty node model falls back
|
|
3085
|
+
* to) mapped to broker slots, checked against the paying person's keys. Throws a
|
|
3086
|
+
* Preflight error naming every missing key; a broker it can't ask never blocks here
|
|
3087
|
+
* (the first spawn reports that instead).
|
|
3088
|
+
*/
|
|
3089
|
+
async _brokerPreflight(manifest, stepModels) {
|
|
3090
|
+
if (!brokerEnabled() || mockEnabled({ mock: this.claude.mock })) return;
|
|
3091
|
+
let info;
|
|
3092
|
+
try { info = await brokerInfo(); } catch { return; }
|
|
3093
|
+
const person = info.mode === 'multi' ? currentBillTo() : 'local';
|
|
3094
|
+
if (info.mode === 'multi' && (!person || person === 'local') && !process.env.WORCA_BROKER_SYSTEM_BILL_TO) {
|
|
3095
|
+
throw Object.assign(new Error('Preflight failed: this run has no signed-in person to charge. Start it from the web UI, or set WORCA_BROKER_SYSTEM_BILL_TO.'), { errorClass: 'auth' });
|
|
3096
|
+
}
|
|
3097
|
+
const models = manifestModels(manifest);
|
|
3098
|
+
for (const m of Object.values(stepModels || {})) if (typeof m === 'string' && m.trim()) models.add(m.trim());
|
|
3099
|
+
if (this.claude?.model) models.add(this.claude.model);
|
|
3100
|
+
if (!models.size) models.add('claude-sonnet-5'); // nothing named: the CLI's own default is a Claude model
|
|
3101
|
+
let status;
|
|
3102
|
+
try { status = (await personSlots(person === 'local' || !person ? (process.env.WORCA_BROKER_SYSTEM_BILL_TO || 'local') : person)).slots || []; } catch { return; }
|
|
3103
|
+
const r = missingCredentials([...models], modelSlot, status);
|
|
3104
|
+
if (r.missing.length || r.errors.length) {
|
|
3105
|
+
throw Object.assign(new Error(`Preflight failed: ${describeMissing(r, info.publicUrl)}`), { errorClass: 'auth' });
|
|
3106
|
+
}
|
|
3107
|
+
}
|
|
3108
|
+
|
|
3026
3109
|
_preflightAgentKeys(agentKeys) {
|
|
3027
3110
|
const reg = this.registry || {};
|
|
3028
3111
|
const missing = [];
|
|
@@ -3154,7 +3237,9 @@ export class RunHarness extends EventEmitter {
|
|
|
3154
3237
|
await appendAudit(this.pipeline.dir, `Recoverable **${cls}** error on ${node.key}: ${firstLine(err.message)}`).catch(() => {});
|
|
3155
3238
|
|
|
3156
3239
|
if (verdict.outcome === 'retry') {
|
|
3157
|
-
|
|
3240
|
+
const delayMs = recoveryDelayMs({ cls, attempt, err });
|
|
3241
|
+
this._log(node.key, 'warn', `${cls}: retrying in ${Math.round(delayMs / 100) / 10}s (retry ${attempt}/${RECOVERY_MAX_AUTO_ATTEMPTS})`);
|
|
3242
|
+
await this._backoff(attempt, this.pauseAbort.signal, { cls, err, delayMs });
|
|
3158
3243
|
return verdict;
|
|
3159
3244
|
}
|
|
3160
3245
|
|
|
@@ -3193,23 +3278,11 @@ export class RunHarness extends EventEmitter {
|
|
|
3193
3278
|
return next;
|
|
3194
3279
|
}
|
|
3195
3280
|
|
|
3196
|
-
/** Abort-aware backoff: base
|
|
3197
|
-
*
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
return Number.isFinite(n) && n >= 0 ? n : 1000;
|
|
3202
|
-
})();
|
|
3203
|
-
const ms = base * Math.pow(2, Math.max(0, attempt - 1));
|
|
3204
|
-
if (!ms) return Promise.resolve();
|
|
3205
|
-
return new Promise((res) => {
|
|
3206
|
-
const t = setTimeout(res, ms);
|
|
3207
|
-
t.unref?.();
|
|
3208
|
-
if (signal) {
|
|
3209
|
-
if (signal.aborted) { clearTimeout(t); res(); }
|
|
3210
|
-
else signal.addEventListener('abort', () => { clearTimeout(t); res(); }, { once: true });
|
|
3211
|
-
}
|
|
3212
|
-
});
|
|
3281
|
+
/** Abort-aware backoff (recovery-backoff.mjs: base·2^(attempt-1), longer for a
|
|
3282
|
+
* rate limit, at least a retry-after hint, capped per wait), resolving early
|
|
3283
|
+
* (and still 'retry') if the pause-only signal fires so a pause is not delayed. */
|
|
3284
|
+
_backoff(attempt, signal, { cls = null, err = null, delayMs } = {}) {
|
|
3285
|
+
return sleepAbortable(delayMs ?? recoveryDelayMs({ cls, attempt, err }), signal);
|
|
3213
3286
|
}
|
|
3214
3287
|
|
|
3215
3288
|
/** Monotonic id source for recovery prompts (no Date.now/random — replay-safe). */
|
|
@@ -3977,9 +4050,16 @@ export class RunHarness extends EventEmitter {
|
|
|
3977
4050
|
// instead of once per node. Looked up ONCE and shared with observeModelCost
|
|
3978
4051
|
// below: modelCostConfig re-reads settings.json on every call.
|
|
3979
4052
|
const costCfg = isResult && attr?.model ? modelCostConfig(attr.model) : null;
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
|
|
4053
|
+
// A bridged node whose upstream reported what its calls cost (OpenRouter's
|
|
4054
|
+
// usage.cost, booked per execution id) records that figure: the CLI prices an
|
|
4055
|
+
// id it does not know at $0, and a pinned price is only an estimate of it.
|
|
4056
|
+
// Read before _recordBridgeCalls forgets the tag.
|
|
4057
|
+
const upstreamCost = isResult && attr?.executionId ? bridgeCostFor(attr.executionId) : null;
|
|
4058
|
+
const cost = upstreamCost
|
|
4059
|
+
? upstreamCost.costUsd
|
|
4060
|
+
: costCfg
|
|
4061
|
+
? resolveModelCost(attr.model, rawCost, e.raw.usage, costCfg)
|
|
4062
|
+
: rawCost;
|
|
3983
4063
|
if (isResult) this._recordBridgeCalls(attr?.stepKey, attr?.executionId);
|
|
3984
4064
|
if (Number.isFinite(cost)) this._recordCost(cost, attr?.stepKey);
|
|
3985
4065
|
else if (isResult && !this.claude.mock) {
|
|
@@ -4406,6 +4486,8 @@ export class RunHarness extends EventEmitter {
|
|
|
4406
4486
|
if (!step) return;
|
|
4407
4487
|
step.bridgeCalls = (step.bridgeCalls || 0) + calls.initiated;
|
|
4408
4488
|
step.bridgeContinued = (step.bridgeContinued || 0) + calls.continued;
|
|
4489
|
+
// OpenRouter `:free` calls (continuations too): what the step spent of the day's allowance.
|
|
4490
|
+
if (calls.free) step.bridgeFreeCalls = (step.bridgeFreeCalls || 0) + calls.free;
|
|
4409
4491
|
this.state.updatedAt = new Date().toISOString();
|
|
4410
4492
|
this._emit('state', this.getState());
|
|
4411
4493
|
this._persist().catch(() => {});
|