orizu 0.5.13 → 0.5.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hosted-boot.js +22 -4
- package/dist/hosted-harness-opencode.js +144 -1
- package/dist/hosted-loop-lifecycle.js +20 -0
- package/dist/hosted-loop.js +96 -6
- package/dist/hosted-run-event-sink.js +91 -18
- package/dist/hosted-session-cli.js +4 -2
- package/package.json +1 -1
- package/src/hosted-boot.ts +26 -4
- package/src/hosted-harness-opencode.ts +204 -1
- package/src/hosted-loop-lifecycle.ts +21 -0
- package/src/hosted-loop.ts +116 -3
- package/src/hosted-run-event-sink.ts +89 -22
- package/src/hosted-session-cli.ts +4 -2
package/dist/hosted-boot.js
CHANGED
|
@@ -33,7 +33,7 @@ import { spawnSync } from 'child_process';
|
|
|
33
33
|
import { closeSync, mkdirSync, openSync, readFileSync, renameSync, writeSync } from 'fs';
|
|
34
34
|
import { AGENT_GIT_IDENTITY, BEARER_BASENAME, BOOT_CONTEXT_BASENAME, DEFAULT_CACHE_REFRESH_BUFFER_MS, HELPER_SCRIPT_BASENAME, REPO_CRED_CACHE_BASENAME, renderCredentialHelperScript, serializeBootContext, } from './hosted-runtime-assets.js';
|
|
35
35
|
import { INJECTED_ENV_VARS_ENV, runHostedLoop, } from './hosted-loop.js';
|
|
36
|
-
import { DEFAULT_EGRESS_CANARY_HOST } from './hosted-loop-lifecycle.js';
|
|
36
|
+
import { DEFAULT_EGRESS_CANARY_HOST, DEFAULT_HOSTED_MODEL } from './hosted-loop-lifecycle.js';
|
|
37
37
|
import { resumeRunEventSink } from './hosted-run-event-sink.js';
|
|
38
38
|
import { stageOrizuSkill } from './hosted-skill-staging.js';
|
|
39
39
|
/** Non-secret placeholder model key (firewall brokers the real org key on
|
|
@@ -42,8 +42,6 @@ import { stageOrizuSkill } from './hosted-skill-staging.js';
|
|
|
42
42
|
* DO-path `workers/session-coordinator/src/bootstrap.ts`); kept in sync by
|
|
43
43
|
* grep. Inlined (not imported) to avoid a hosted-session-cli import cycle. */
|
|
44
44
|
const ANTHROPIC_DUMMY_KEY = 'sk-ant-orizu-proxy-broker-placeholder';
|
|
45
|
-
/** Default hosted model — mirrors hosted-session-cli.ts `DEFAULT_HOSTED_MODEL`. */
|
|
46
|
-
const DEFAULT_HOSTED_MODEL = 'anthropic/claude-opus-4-8';
|
|
47
45
|
// -- Frozen env contract ------------------------------------------------------
|
|
48
46
|
export const REQUIRED_BOOT_ENV_VARS = [
|
|
49
47
|
'ORIZU_BOOT_SECRET',
|
|
@@ -371,6 +369,16 @@ export async function runHostedBoot(opts) {
|
|
|
371
369
|
const readFile = opts.readFile ?? ((path) => readFileSync(path, 'utf8'));
|
|
372
370
|
const log = opts.log ?? (() => { });
|
|
373
371
|
const now = opts.now ?? (() => Date.now());
|
|
372
|
+
// 0 — ENV HYGIENE (ALI-1062): the boot secret is the DO path's only durable
|
|
373
|
+
// credential (it mints agent bearers at the internet-reachable agent-token
|
|
374
|
+
// route for up to 24h), and `env.bootSecret` is already captured — so scrub
|
|
375
|
+
// the raw value from the process env NOW, before anything downstream can
|
|
376
|
+
// inherit it: the opencode/agent process is spawned with `{...process.env}`
|
|
377
|
+
// (nodeChildSpawner) and the customer `.orizu/setup.sh` hook inherits the
|
|
378
|
+
// env too. Nothing reads it from the env after this point — the rotation /
|
|
379
|
+
// boot-status closures hold `env.bootSecret`, and the git credential helper
|
|
380
|
+
// reads the 0600 run-dir boot-secret FILE (written in step 4).
|
|
381
|
+
delete processEnv.ORIZU_BOOT_SECRET;
|
|
374
382
|
// 1 — Pull the agent bearer (retry/backoff — the DO may still be arming).
|
|
375
383
|
const bearer = await pullAgentBearer({
|
|
376
384
|
agentTokenUrl: env.agentTokenUrl,
|
|
@@ -533,10 +541,20 @@ export async function runHostedBoot(opts) {
|
|
|
533
541
|
});
|
|
534
542
|
try {
|
|
535
543
|
const runLoop = opts.runLoop ??
|
|
536
|
-
((input) => runHostedLoop({
|
|
544
|
+
((input) => runHostedLoop({
|
|
545
|
+
context: input.context,
|
|
546
|
+
taskPrompt: input.taskPrompt,
|
|
547
|
+
bearerProvider: input.bearerProvider,
|
|
548
|
+
redactSecretsList: input.redactSecretsList,
|
|
549
|
+
}));
|
|
537
550
|
const result = await runLoop({
|
|
538
551
|
context: loopContext,
|
|
539
552
|
taskPrompt: session.task,
|
|
553
|
+
// ALI-1062 exact-value redaction: the boot secret is 64 bare hex chars —
|
|
554
|
+
// no TOKEN_SHAPE_PATTERNS rule can catch it, so it MUST ride the loop's
|
|
555
|
+
// verbatim redaction list or an echoed env dump would land it in
|
|
556
|
+
// workbench_run_events.
|
|
557
|
+
redactSecretsList: [env.bootSecret],
|
|
540
558
|
// Read the bearer FILE fresh per request (review F1): `startBearerRotation`
|
|
541
559
|
// rewrites `bearerFileAbs` before the current bearer expires, so a run
|
|
542
560
|
// exceeding the ~60-min TTL (DO cap is 24h) keeps a valid bearer. A frozen
|
|
@@ -120,6 +120,22 @@ const ANTHROPIC_ADAPTIVE_THINKING_MODELS = new Set([
|
|
|
120
120
|
'claude-sonnet-4-6',
|
|
121
121
|
]);
|
|
122
122
|
const ANTHROPIC_ADAPTIVE_EFFORTS = new Set(['low', 'medium', 'high', 'xhigh', 'max']);
|
|
123
|
+
/**
|
|
124
|
+
* Split a possibly provider-qualified model string into provider + model id.
|
|
125
|
+
* SINGLE-SLASH ids ("anthropic/claude-opus-4-8") are the assumed form; a bare
|
|
126
|
+
* id defaults to provider "anthropic". This is THE shared parse — consumed by
|
|
127
|
+
* BOTH the prompt path (`buildPromptRequestBody`) and the pre-prompt
|
|
128
|
+
* validation (`awaitOpenCodeModelResolvable`), so the two can never drift:
|
|
129
|
+
* parity is exactly what makes the validation meaningful (ALI-1086).
|
|
130
|
+
* KNOWN LIMIT: `split('/', 2)` drops everything after the second slash, so a
|
|
131
|
+
* 3-part id ("openrouter/anthropic/claude-x") misparses to provider
|
|
132
|
+
* "openrouter" + model "anthropic" — identically in both places (today's
|
|
133
|
+
* pre-existing prompt-path behavior, now shared rather than duplicated).
|
|
134
|
+
*/
|
|
135
|
+
export function qualifyHostedModel(model) {
|
|
136
|
+
const [providerId, modelId] = model.includes('/') ? model.split('/', 2) : ['anthropic', model];
|
|
137
|
+
return { providerId, modelId };
|
|
138
|
+
}
|
|
123
139
|
export function buildPromptRequestBody(content, model, opencodeMessageId, reasoningEffort) {
|
|
124
140
|
const body = {
|
|
125
141
|
parts: [{ type: 'text', text: content }],
|
|
@@ -127,7 +143,9 @@ export function buildPromptRequestBody(content, model, opencodeMessageId, reason
|
|
|
127
143
|
};
|
|
128
144
|
if (!model)
|
|
129
145
|
return body;
|
|
130
|
-
|
|
146
|
+
// Shared parse (see qualifyHostedModel): keeps the prompt path and the
|
|
147
|
+
// ALI-1086 pre-prompt validation in lockstep by construction.
|
|
148
|
+
const { providerId, modelId } = qualifyHostedModel(model);
|
|
131
149
|
const modelSpec = { providerID: providerId, modelID: modelId };
|
|
132
150
|
if (reasoningEffort) {
|
|
133
151
|
if (providerId === 'anthropic') {
|
|
@@ -688,6 +706,131 @@ function errorEvent(messageId, error) {
|
|
|
688
706
|
const message = error instanceof Error ? error.message : String(error);
|
|
689
707
|
return { kind: 'error', messageId, critical: true, payload: { error: message } };
|
|
690
708
|
}
|
|
709
|
+
const MODEL_CATALOG_REQUEST_TIMEOUT_MS = 10_000;
|
|
710
|
+
/**
|
|
711
|
+
* Read the resolvable catalog from a running `opencode serve` via
|
|
712
|
+
* `GET /config/providers` (shape: `{ providers: [{ id, models: { <id>: … } }] }`,
|
|
713
|
+
* verified against opencode-ai@1.14.41). Returns null on ANY transport/shape
|
|
714
|
+
* failure — the caller treats that as "cannot validate", never as "invalid".
|
|
715
|
+
*/
|
|
716
|
+
export async function fetchOpenCodeModelCatalog(baseUrl, fetchImpl = globalThis.fetch, timeoutMs = MODEL_CATALOG_REQUEST_TIMEOUT_MS) {
|
|
717
|
+
const controller = new AbortController();
|
|
718
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
719
|
+
try {
|
|
720
|
+
const res = await fetchImpl(`${baseUrl.replace(/\/$/, '')}/config/providers`, {
|
|
721
|
+
signal: controller.signal,
|
|
722
|
+
});
|
|
723
|
+
if (!res.ok)
|
|
724
|
+
return null;
|
|
725
|
+
const data = asRecord(await res.json());
|
|
726
|
+
if (!Array.isArray(data.providers))
|
|
727
|
+
return null;
|
|
728
|
+
const catalog = new Map();
|
|
729
|
+
for (const entry of data.providers) {
|
|
730
|
+
const provider = asRecord(entry);
|
|
731
|
+
const id = typeof provider.id === 'string' ? provider.id : null;
|
|
732
|
+
if (!id)
|
|
733
|
+
continue;
|
|
734
|
+
catalog.set(id, Object.keys(asRecord(provider.models)));
|
|
735
|
+
}
|
|
736
|
+
return catalog;
|
|
737
|
+
}
|
|
738
|
+
catch {
|
|
739
|
+
return null;
|
|
740
|
+
}
|
|
741
|
+
finally {
|
|
742
|
+
clearTimeout(timer);
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
function sharedPrefixLength(a, b) {
|
|
746
|
+
const max = Math.min(a.length, b.length);
|
|
747
|
+
let i = 0;
|
|
748
|
+
while (i < max && a[i] === b[i])
|
|
749
|
+
i += 1;
|
|
750
|
+
return i;
|
|
751
|
+
}
|
|
752
|
+
/** Rank candidate model ids by similarity to the requested id (longest shared
|
|
753
|
+
* prefix first, ties broken lexicographically DESCENDING so the newest of a
|
|
754
|
+
* versioned family leads) and cap the list — the fail-fast error should lead
|
|
755
|
+
* with the closest/newest substitutes, not a full catalog dump. */
|
|
756
|
+
export function rankModelAlternatives(requested, candidates, limit = 8) {
|
|
757
|
+
return [...new Set(candidates)]
|
|
758
|
+
.sort((a, b) => {
|
|
759
|
+
const byPrefix = sharedPrefixLength(requested, b) - sharedPrefixLength(requested, a);
|
|
760
|
+
return byPrefix !== 0 ? byPrefix : b.localeCompare(a);
|
|
761
|
+
})
|
|
762
|
+
.slice(0, limit);
|
|
763
|
+
}
|
|
764
|
+
const MODEL_RESOLVE_TIMEOUT_MS = 30_000;
|
|
765
|
+
const MODEL_RESOLVE_POLL_MS = 2_000;
|
|
766
|
+
/**
|
|
767
|
+
* Decide whether the requested model is resolvable by the RUNNING opencode,
|
|
768
|
+
* polling briefly so the boot-time models.dev refresh has time to land.
|
|
769
|
+
* Fail-open on validator infrastructure, with a DELIBERATE asymmetry on null
|
|
770
|
+
* catalog reads:
|
|
771
|
+
* - null BEFORE any successful read → `skipped` immediately (dead/absent
|
|
772
|
+
* endpoint; opencode's own `session.error` remains the backstop, and the
|
|
773
|
+
* early exit keeps absent-endpoint runs — and tests — cheap);
|
|
774
|
+
* - null AFTER at least one successful read → a transient boot-window blip
|
|
775
|
+
* (5xx/drop): sleep and keep polling within the budget; a budget that
|
|
776
|
+
* ends in that state returns `skipped`, NEVER a false `unresolvable` off
|
|
777
|
+
* a flaky read.
|
|
778
|
+
* Only a HEALTHY catalog that never lists the model within the budget yields
|
|
779
|
+
* `unresolvable`, with ranked, provider-qualified alternatives.
|
|
780
|
+
*/
|
|
781
|
+
export async function awaitOpenCodeModelResolvable(opts) {
|
|
782
|
+
const timeoutMs = opts.timeoutMs ?? MODEL_RESOLVE_TIMEOUT_MS;
|
|
783
|
+
const pollMs = opts.pollMs ?? MODEL_RESOLVE_POLL_MS;
|
|
784
|
+
const sleep = opts.sleep ?? ((ms) => new Promise(r => setTimeout(r, ms)));
|
|
785
|
+
const now = opts.now ?? (() => Date.now());
|
|
786
|
+
const { providerId, modelId } = qualifyHostedModel(opts.model);
|
|
787
|
+
const requested = `${providerId}/${modelId}`;
|
|
788
|
+
const startedAt = now();
|
|
789
|
+
let lastGoodCatalog = null;
|
|
790
|
+
let sawTransientFailure = false;
|
|
791
|
+
for (;;) {
|
|
792
|
+
const catalog = await fetchOpenCodeModelCatalog(opts.baseUrl, opts.fetchImpl);
|
|
793
|
+
if (catalog === null) {
|
|
794
|
+
if (lastGoodCatalog === null) {
|
|
795
|
+
// Never answered — the endpoint is dead/absent, not blipping. Skip
|
|
796
|
+
// immediately rather than invent a failure mode (or a slow wait) the
|
|
797
|
+
// run would not otherwise have.
|
|
798
|
+
return { kind: 'skipped', reason: 'opencode /config/providers unavailable' };
|
|
799
|
+
}
|
|
800
|
+
// Transient failure after a successful read (boot-window 5xx/drop) —
|
|
801
|
+
// keep polling within the budget.
|
|
802
|
+
sawTransientFailure = true;
|
|
803
|
+
}
|
|
804
|
+
else {
|
|
805
|
+
lastGoodCatalog = catalog;
|
|
806
|
+
sawTransientFailure = false;
|
|
807
|
+
if ((catalog.get(providerId) ?? []).includes(modelId)) {
|
|
808
|
+
return { kind: 'resolvable', waitedMs: now() - startedAt };
|
|
809
|
+
}
|
|
810
|
+
// Deliberately NOT failing fast when the provider itself is unknown:
|
|
811
|
+
// the models.dev refresh can add whole providers too, so the poll runs
|
|
812
|
+
// the full budget either way.
|
|
813
|
+
}
|
|
814
|
+
if (now() - startedAt >= timeoutMs)
|
|
815
|
+
break;
|
|
816
|
+
await sleep(pollMs);
|
|
817
|
+
}
|
|
818
|
+
if (sawTransientFailure || lastGoodCatalog === null) {
|
|
819
|
+
// The budget ended on a flaky read — never declare `unresolvable` off it.
|
|
820
|
+
return { kind: 'skipped', reason: 'opencode /config/providers became unavailable mid-poll' };
|
|
821
|
+
}
|
|
822
|
+
const providerKnown = lastGoodCatalog.has(providerId);
|
|
823
|
+
const candidates = providerKnown
|
|
824
|
+
? (lastGoodCatalog.get(providerId) ?? []).map(id => `${providerId}/${id}`)
|
|
825
|
+
: [...lastGoodCatalog.entries()].flatMap(([pid, ids]) => ids.map(id => `${pid}/${id}`));
|
|
826
|
+
return {
|
|
827
|
+
kind: 'unresolvable',
|
|
828
|
+
requested,
|
|
829
|
+
alternatives: rankModelAlternatives(requested, candidates),
|
|
830
|
+
providerKnown,
|
|
831
|
+
waitedMs: now() - startedAt,
|
|
832
|
+
};
|
|
833
|
+
}
|
|
691
834
|
/**
|
|
692
835
|
* Tools DENIED in a hosted headless run (ALI-1037). OpenCode exposes a `question`
|
|
693
836
|
* permission key (docs/permissions) that gates the interactive "ask the user a
|
|
@@ -21,6 +21,26 @@
|
|
|
21
21
|
* `hosted-loop.ts` re-exports everything here, so the CLI's import surface
|
|
22
22
|
* (`@/packages/cli/src/hosted-loop`) is unchanged.
|
|
23
23
|
*/
|
|
24
|
+
/**
|
|
25
|
+
* Default provider-qualified hosted model — the SINGLE source of truth
|
|
26
|
+
* (ALI-1086). Imported by both the operator-path session CLI
|
|
27
|
+
* (`hosted-session-cli.ts`) and the DO-path boot (`hosted-boot.ts`); never
|
|
28
|
+
* re-inline it.
|
|
29
|
+
*
|
|
30
|
+
* PIN CONSTRAINT: the hosted OpenCode runtime is HARD-PINNED to
|
|
31
|
+
* `opencode-ai@1.14.41` (`OPENCODE_PINNED_VERSION`, SSE-fragile — do not bump)
|
|
32
|
+
* whose BUNDLED model catalog predates this model (newest bundled anthropic
|
|
33
|
+
* opus: claude-opus-4-7). Verified empirically on ALI-1086: 1.14.41 DOES fetch
|
|
34
|
+
* `https://models.dev/api.json` at boot when reachable (cached to
|
|
35
|
+
* `~/.cache/opencode/models.json`, refreshed when stale, disabled only by
|
|
36
|
+
* `OPENCODE_DISABLE_MODELS_FETCH`), so with models.dev on the sandbox egress
|
|
37
|
+
* allowlist (#1392, `ORIZU_HOSTED_EGRESS_ALLOWLIST`) this id resolves in new
|
|
38
|
+
* sandboxes. If the catalog fetch is blocked/stale anyway, the loop's
|
|
39
|
+
* pre-prompt validation (`awaitOpenCodeModelResolvable`) fails the run fast,
|
|
40
|
+
* naming the resolvable alternatives. The durable fix for the stale bundled
|
|
41
|
+
* catalog is the ALI-929 harness swap.
|
|
42
|
+
*/
|
|
43
|
+
export const DEFAULT_HOSTED_MODEL = 'anthropic/claude-opus-4-8';
|
|
24
44
|
/**
|
|
25
45
|
* Environment injected into the spawned OpenCode server so the AGENT's bash sees a
|
|
26
46
|
* PRE-AUTHENTICATED `orizu` CLI (ALI-1044). ORIZU_TOKEN_FILE points at the 0600
|
package/dist/hosted-loop.js
CHANGED
|
@@ -39,14 +39,14 @@ import { join } from 'path';
|
|
|
39
39
|
import { HOSTED_TASK_PREAMBLE, PREBAKED_MARKER_PATH, SETUP_HOOK_RELATIVE_PATH, composeHostedTaskPrompt, parsePrebakedMarker, } from './hosted-runtime-assets.js';
|
|
40
40
|
import { harvestWorkspace } from './hosted-harvest.js';
|
|
41
41
|
import { annotateHeadlessQuestions, withIdleWatchdog } from './hosted-headless.js';
|
|
42
|
-
import { DEFAULT_PROMPT_MAX_DURATION_MS, OPENCODE_PINNED_VERSION, createOpenCodeHarness, spawnOpenCode, } from './hosted-harness-opencode.js';
|
|
42
|
+
import { DEFAULT_PROMPT_MAX_DURATION_MS, OPENCODE_PINNED_VERSION, awaitOpenCodeModelResolvable, createOpenCodeHarness, spawnOpenCode, } from './hosted-harness-opencode.js';
|
|
43
43
|
import { createClaudeAgentHarness } from './hosted-harness-claude.js';
|
|
44
|
-
import { drainHarnessToSink, resumeRunEventSink, } from './hosted-run-event-sink.js';
|
|
44
|
+
import { drainHarnessToSink, resumeRunEventSink, RunTerminalError, TerminalDeliveryError, } from './hosted-run-event-sink.js';
|
|
45
45
|
import { buildOpenCodeSpawnEnv, defaultProbeEgress, egressCanaryAllowedHost, runEgressCanary, } from './hosted-loop-lifecycle.js';
|
|
46
46
|
// The lifecycle/orchestration seam moved to `hosted-loop-lifecycle.ts`
|
|
47
47
|
// (ALI-1015 Phase 1) so `lib/hosted-runtime/` can share it with a server-side
|
|
48
48
|
// coordinator. Re-exported here so this module's public surface is unchanged.
|
|
49
|
-
export { DEFAULT_EGRESS_CANARY_HOST, buildOpenCodeSpawnEnv, egressCanaryAllowedHost, runEgressCanary, } from './hosted-loop-lifecycle.js';
|
|
49
|
+
export { DEFAULT_EGRESS_CANARY_HOST, DEFAULT_HOSTED_MODEL, buildOpenCodeSpawnEnv, egressCanaryAllowedHost, runEgressCanary, } from './hosted-loop-lifecycle.js';
|
|
50
50
|
// Time reserved BEFORE the sandbox self-terminates so a duration-capped prompt
|
|
51
51
|
// abort still has room to harvest partial work (commit/push) + write the terminal
|
|
52
52
|
// event: `promptMaxDurationMs = sandboxBudgetMs − this` (ALI-1061). 5 min is
|
|
@@ -97,6 +97,15 @@ function resolveIdleTimeoutMs(override) {
|
|
|
97
97
|
}
|
|
98
98
|
return DEFAULT_AGENT_IDLE_TIMEOUT_MS;
|
|
99
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* Loop-level retry schedule for an undeliverable TERMINAL transition (ALI-1065
|
|
102
|
+
* finding 2). Each entry is a pause before one more full sink delivery cycle
|
|
103
|
+
* (which itself retries ~4 times over ~350ms), so a transport blip of a few
|
|
104
|
+
* seconds recovers with the ORIGINAL status instead of being re-recorded as
|
|
105
|
+
* 'failed'. Bounded: a persistently unreachable control plane fails the
|
|
106
|
+
* DELIVERY visibly (result.error) without recording a wrong outcome.
|
|
107
|
+
*/
|
|
108
|
+
export const TERMINAL_DELIVERY_RETRY_DELAYS_MS = [2_000, 8_000];
|
|
100
109
|
/** Model-key env vars whose value must be scrubbed from every run event. */
|
|
101
110
|
const MODEL_KEY_ENV_VARS = ['ANTHROPIC_API_KEY', 'OPENAI_API_KEY'];
|
|
102
111
|
/**
|
|
@@ -399,6 +408,40 @@ export async function runHostedLoop(opts) {
|
|
|
399
408
|
payload: { step: 'opencode_ready', ok: true, waitedMs: spawned.readyAfterMs },
|
|
400
409
|
});
|
|
401
410
|
}
|
|
411
|
+
// Fail-fast model validation (ALI-1086): ask the RUNNING opencode whether
|
|
412
|
+
// the requested model is resolvable BEFORE the first prompt. The pinned
|
|
413
|
+
// opencode's bundled catalog is stale (predates claude-opus-4-8) and only
|
|
414
|
+
// a boot-time models.dev fetch (#1392 allowlists it) refreshes it — when
|
|
415
|
+
// that fetch is blocked, the first prompt used to die inside the SSE
|
|
416
|
+
// stream with an opaque `run_failed: Model not found`. Now the run fails
|
|
417
|
+
// immediately with a structured artifact + an error naming the requested
|
|
418
|
+
// id AND the resolvable alternatives. Fail-open on validator
|
|
419
|
+
// infrastructure: an unreachable catalog endpoint (`skipped`) proceeds
|
|
420
|
+
// silently — opencode's own session.error remains the backstop. Healthy
|
|
421
|
+
// and skipped runs append NO event, so the ALI-929 cross-harness
|
|
422
|
+
// stream-identity invariant is untouched (validation exists only on the
|
|
423
|
+
// opencode path; the Claude-SDK driver resolves models itself).
|
|
424
|
+
const validateModel = opts.validateModel ?? awaitOpenCodeModelResolvable;
|
|
425
|
+
const validation = await validateModel({ baseUrl: spawned.baseUrl, model: context.model });
|
|
426
|
+
if (validation.kind === 'unresolvable') {
|
|
427
|
+
await sink.append({
|
|
428
|
+
kind: 'artifact',
|
|
429
|
+
payload: {
|
|
430
|
+
step: 'model_validation',
|
|
431
|
+
ok: false,
|
|
432
|
+
requestedModel: validation.requested,
|
|
433
|
+
resolvableAlternatives: validation.alternatives,
|
|
434
|
+
providerKnown: validation.providerKnown,
|
|
435
|
+
opencodeVersion: pinnedVersion,
|
|
436
|
+
waitedMs: validation.waitedMs,
|
|
437
|
+
},
|
|
438
|
+
});
|
|
439
|
+
throw new Error(`requested model ${validation.requested} is not resolvable by the pinned ` +
|
|
440
|
+
`opencode runtime (opencode-ai@${pinnedVersion}); resolvable alternatives: ` +
|
|
441
|
+
`${validation.alternatives.join(', ') || '(none)'} — retry with --model ` +
|
|
442
|
+
`set to one of these (stale bundled catalog + blocked models.dev refresh; ` +
|
|
443
|
+
`see ALI-1086, durable fix ALI-929)`);
|
|
444
|
+
}
|
|
402
445
|
// Derive the per-prompt max-duration cap from the sandbox budget so a long
|
|
403
446
|
// `--duration` run is not killed at the hard-coded 90-min cap while its
|
|
404
447
|
// sandbox is still alive; floored at the harness default for short/undated
|
|
@@ -472,10 +515,57 @@ export async function runHostedLoop(opts) {
|
|
|
472
515
|
}
|
|
473
516
|
catch (error) {
|
|
474
517
|
const message = error instanceof Error ? error.message : String(error);
|
|
475
|
-
//
|
|
476
|
-
//
|
|
477
|
-
// is valuable, incl. an idle-watchdog abort) — before the terminal seals.
|
|
518
|
+
// Harvest FIRST on every abnormal path (partial work is valuable, incl. an
|
|
519
|
+
// idle-watchdog abort) — before any terminal write seals the sink.
|
|
478
520
|
await runAutoHarvest();
|
|
521
|
+
// ALI-1065 finding 2: an undeliverable TERMINAL transition must NEVER be
|
|
522
|
+
// re-recorded as 'failed' — the run's outcome was already decided; only its
|
|
523
|
+
// DELIVERY failed. Retry the delivery with the ORIGINAL status (spaced past
|
|
524
|
+
// the sink's own ~350ms retry window, so a ~1s transport blip recovers),
|
|
525
|
+
// and if it still cannot land, fail the DELIVERY visibly: return the
|
|
526
|
+
// intended status with a non-null error, which the boot reports via its
|
|
527
|
+
// boot-status callback, and the coordinator's session-end finalizer sweeps
|
|
528
|
+
// the still-'running' row server-side with an explicit marker.
|
|
529
|
+
if (error instanceof TerminalDeliveryError) {
|
|
530
|
+
const sleep = opts.sleepImpl ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)));
|
|
531
|
+
for (const delayMs of TERMINAL_DELIVERY_RETRY_DELAYS_MS) {
|
|
532
|
+
if (sink.sealed)
|
|
533
|
+
break;
|
|
534
|
+
await sleep(delayMs);
|
|
535
|
+
try {
|
|
536
|
+
await sink.finish(error.intendedStatus, error.finishOptions);
|
|
537
|
+
return { status: error.intendedStatus, agentSessionId, installOk, error: null };
|
|
538
|
+
}
|
|
539
|
+
catch (retryError) {
|
|
540
|
+
// The run went terminal server-side while we were retrying: accept
|
|
541
|
+
// the server record (Orizu records win) instead of retrying further.
|
|
542
|
+
if (retryError instanceof RunTerminalError) {
|
|
543
|
+
return { status: 'cancelled', agentSessionId, installOk, error: null };
|
|
544
|
+
}
|
|
545
|
+
// Otherwise keep retrying within the bounded schedule; never change
|
|
546
|
+
// the status.
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
if (sink.sealed) {
|
|
550
|
+
// A concurrent/preceding delivery landed after all — the record is
|
|
551
|
+
// terminal server-side with our intended status semantics preserved.
|
|
552
|
+
return { status: error.intendedStatus, agentSessionId, installOk, error: null };
|
|
553
|
+
}
|
|
554
|
+
return {
|
|
555
|
+
status: error.intendedStatus,
|
|
556
|
+
agentSessionId,
|
|
557
|
+
installOk,
|
|
558
|
+
error: `terminal delivery failed (intended status preserved): ${message}`,
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
// The run went terminal SERVER-side (cancel/finish out from under this
|
|
562
|
+
// writer): the sink sealed itself; accept the server record (Orizu records
|
|
563
|
+
// win) instead of fabricating a local failure.
|
|
564
|
+
if (error instanceof RunTerminalError) {
|
|
565
|
+
return { status: 'cancelled', agentSessionId, installOk, error: null };
|
|
566
|
+
}
|
|
567
|
+
// Fail closed: never leave the run non-terminal. finish() is idempotent and
|
|
568
|
+
// no-ops when the drain already sealed the sink.
|
|
479
569
|
if (!sink.sealed) {
|
|
480
570
|
try {
|
|
481
571
|
await sink.finish('failed', { summary: { error: message } });
|
|
@@ -115,6 +115,26 @@ export class CriticalDeliveryError extends Error {
|
|
|
115
115
|
this.name = 'CriticalDeliveryError';
|
|
116
116
|
}
|
|
117
117
|
}
|
|
118
|
+
/**
|
|
119
|
+
* Raised when `finish()` itself is undeliverable (ALI-1065 finding 2): a
|
|
120
|
+
* CriticalDeliveryError thrown WHILE DELIVERING A TERMINAL TRANSITION, carrying
|
|
121
|
+
* the intended status + finish options so the caller can retry the delivery
|
|
122
|
+
* with the ORIGINAL outcome. Before this, the error surfaced as a bare
|
|
123
|
+
* CriticalDeliveryError and the loop's generic catch re-finished the run
|
|
124
|
+
* 'failed' — a ~1s transport blip on the SUCCESS terminal recorded a succeeded
|
|
125
|
+
* run as failed. Subclasses CriticalDeliveryError so existing classification
|
|
126
|
+
* (drop-vs-surface) is unchanged.
|
|
127
|
+
*/
|
|
128
|
+
export class TerminalDeliveryError extends CriticalDeliveryError {
|
|
129
|
+
intendedStatus;
|
|
130
|
+
finishOptions;
|
|
131
|
+
constructor(message, intendedStatus, finishOptions) {
|
|
132
|
+
super(message);
|
|
133
|
+
this.name = 'TerminalDeliveryError';
|
|
134
|
+
this.intendedStatus = intendedStatus;
|
|
135
|
+
this.finishOptions = finishOptions;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
118
138
|
/**
|
|
119
139
|
* Raised on a 403 from the RunAPI. Named for what a 403 actually is — NOT "the
|
|
120
140
|
* run is terminal" (that is 404/410). A 403 means the caller may no longer write
|
|
@@ -423,13 +443,28 @@ export function createRunEventSink(options) {
|
|
|
423
443
|
diag(`finish(${status}) called on an already-sealed sink — ignored (idempotent)`);
|
|
424
444
|
return;
|
|
425
445
|
}
|
|
426
|
-
//
|
|
427
|
-
//
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
446
|
+
// ALI-1065 finding 2: a delivery failure ANYWHERE inside finish (the
|
|
447
|
+
// transcript tail is a critical append; the terminal PATCH is the
|
|
448
|
+
// transition itself) must carry the INTENDED status out to the caller.
|
|
449
|
+
// Without this, the loop's generic catch converted an undeliverable
|
|
450
|
+
// SUCCESS terminal into a recorded 'failed'. RunTerminalError/RunAuthError
|
|
451
|
+
// pass through unchanged — they are classifications, not delivery
|
|
452
|
+
// failures.
|
|
453
|
+
try {
|
|
454
|
+
// Pre-terminal appends (still writable): flush coalesced tokens, then the
|
|
455
|
+
// bounded transcript tail (the seal below refuses any later append).
|
|
456
|
+
await flushTokens();
|
|
457
|
+
if (opts?.transcript) {
|
|
458
|
+
await postEvent(AGENT_TRANSCRIPT_EVENT_TYPE, { text: tail(opts.transcript, TRANSCRIPT_TAIL_MAX_CHARS), truncated: opts.transcript.length > TRANSCRIPT_TAIL_MAX_CHARS }, true);
|
|
459
|
+
}
|
|
460
|
+
await patchTerminal(status, opts);
|
|
461
|
+
}
|
|
462
|
+
catch (error) {
|
|
463
|
+
if (error instanceof CriticalDeliveryError && !(error instanceof TerminalDeliveryError)) {
|
|
464
|
+
throw new TerminalDeliveryError(error.message, status, opts);
|
|
465
|
+
}
|
|
466
|
+
throw error;
|
|
431
467
|
}
|
|
432
|
-
await patchTerminal(status, opts);
|
|
433
468
|
}
|
|
434
469
|
return {
|
|
435
470
|
append,
|
|
@@ -513,6 +548,42 @@ export async function drainHarnessToSink(events, sink, opts = {}) {
|
|
|
513
548
|
if (opts.beforeFinish)
|
|
514
549
|
await opts.beforeFinish();
|
|
515
550
|
};
|
|
551
|
+
// ALI-1065 finding 3: the server now rejects appends to a terminal run with
|
|
552
|
+
// 410 (the run was cancelled/finished out from under this writer). The sink
|
|
553
|
+
// seals itself and raises RunTerminalError; the drain must treat that as a
|
|
554
|
+
// clean server-side termination — stop consuming and report 'cancelled' —
|
|
555
|
+
// not crash the loop into a spurious re-finish (Orizu records win; the
|
|
556
|
+
// stored status is whatever the server committed). The hook still runs so
|
|
557
|
+
// partial work is harvested, but with a sealed sink its own appends no-op.
|
|
558
|
+
const handleServerTerminal = async () => {
|
|
559
|
+
await runBeforeFinish();
|
|
560
|
+
return 'cancelled';
|
|
561
|
+
};
|
|
562
|
+
// finish() can ALSO surface RunTerminalError (its pre-terminal token flush /
|
|
563
|
+
// transcript append hits the 410 before the PATCH ever runs) — accept the
|
|
564
|
+
// server's termination the same way instead of crashing the drain. A finish
|
|
565
|
+
// whose PATCH meets an already-terminal run does NOT throw (patchTerminal
|
|
566
|
+
// seals on a terminal 409), so this wrapper changes nothing on that path.
|
|
567
|
+
//
|
|
568
|
+
// SEALED-BEFORE-FINISH: the beforeFinish hook's own best-effort appends
|
|
569
|
+
// (auto-harvest events) swallow errors — including the RunTerminalError of a
|
|
570
|
+
// 410 that SEALED the sink. finish() on a sealed sink is an idempotent no-op,
|
|
571
|
+
// so without this check the drain would report the intended status as
|
|
572
|
+
// delivered when in fact the server terminated the run out from under us.
|
|
573
|
+
const finishAccepting = async (status, finishOpts) => {
|
|
574
|
+
await runBeforeFinish();
|
|
575
|
+
if (sink.sealed)
|
|
576
|
+
return 'cancelled';
|
|
577
|
+
try {
|
|
578
|
+
await sink.finish(status, finishOpts);
|
|
579
|
+
return status;
|
|
580
|
+
}
|
|
581
|
+
catch (error) {
|
|
582
|
+
if (error instanceof RunTerminalError)
|
|
583
|
+
return 'cancelled';
|
|
584
|
+
throw error;
|
|
585
|
+
}
|
|
586
|
+
};
|
|
516
587
|
for await (const event of events) {
|
|
517
588
|
if (event.kind === 'execution_complete') {
|
|
518
589
|
// A prompt the caller/agent stopped is a cancellation, not a failure: the
|
|
@@ -520,40 +591,42 @@ export async function drainHarnessToSink(events, sink, opts = {}) {
|
|
|
520
591
|
// and stop()-initiated paths. A genuine failure (success:false, no abort
|
|
521
592
|
// flag) still lands 'failed'.
|
|
522
593
|
if (event.payload.aborted === true) {
|
|
523
|
-
|
|
524
|
-
await sink.finish('cancelled', {
|
|
594
|
+
return finishAccepting('cancelled', {
|
|
525
595
|
summary: opts.summary,
|
|
526
596
|
evidence: opts.evidence,
|
|
527
597
|
transcript: opts.transcript,
|
|
528
598
|
});
|
|
529
|
-
return 'cancelled';
|
|
530
599
|
}
|
|
531
600
|
const success = event.payload.success !== false;
|
|
532
601
|
const status = success ? 'succeeded' : 'failed';
|
|
533
602
|
const summary = success
|
|
534
603
|
? opts.summary
|
|
535
604
|
: { ...(opts.summary ?? {}), error: event.payload.error ?? 'execution failed' };
|
|
536
|
-
|
|
537
|
-
await sink.finish(status, { summary, evidence: opts.evidence, transcript: opts.transcript });
|
|
538
|
-
return status;
|
|
605
|
+
return finishAccepting(status, { summary, evidence: opts.evidence, transcript: opts.transcript });
|
|
539
606
|
}
|
|
540
607
|
if (event.kind === 'error') {
|
|
541
|
-
|
|
542
|
-
await sink.finish('failed', {
|
|
608
|
+
return finishAccepting('failed', {
|
|
543
609
|
summary: { ...(opts.summary ?? {}), error: event.payload.error ?? 'agent error' },
|
|
544
610
|
evidence: opts.evidence,
|
|
545
611
|
transcript: opts.transcript,
|
|
546
612
|
});
|
|
547
|
-
return 'failed';
|
|
548
613
|
}
|
|
549
|
-
|
|
614
|
+
try {
|
|
615
|
+
await sink.append(event);
|
|
616
|
+
}
|
|
617
|
+
catch (error) {
|
|
618
|
+
if (error instanceof RunTerminalError)
|
|
619
|
+
return handleServerTerminal();
|
|
620
|
+
throw error;
|
|
621
|
+
}
|
|
550
622
|
}
|
|
551
623
|
// The harness contract guarantees a terminal event; reaching here means the
|
|
552
624
|
// stream ended abnormally. Fail closed so the run never dangles non-terminal.
|
|
553
625
|
await runBeforeFinish();
|
|
554
|
-
|
|
626
|
+
if (sink.sealed)
|
|
627
|
+
return 'cancelled';
|
|
628
|
+
return finishAccepting('failed', {
|
|
555
629
|
summary: { ...(opts.summary ?? {}), error: 'harness stream ended without a terminal event' },
|
|
556
630
|
transcript: opts.transcript,
|
|
557
631
|
});
|
|
558
|
-
return 'failed';
|
|
559
632
|
}
|
|
@@ -40,7 +40,7 @@ import { resumeRunEventSink } from './hosted-run-event-sink.js';
|
|
|
40
40
|
import { AGENT_GIT_IDENTITY, BEARER_BASENAME } from './hosted-runtime-assets.js';
|
|
41
41
|
import { createVercelProvider } from './vercel-sandbox-provider.js';
|
|
42
42
|
import { buildEgressPolicy, DEFAULT_EGRESS_CANARY_HOST } from './egress-policy.js';
|
|
43
|
-
import { hostedLoopCommand } from './hosted-loop.js';
|
|
43
|
+
import { DEFAULT_HOSTED_MODEL, hostedLoopCommand } from './hosted-loop.js';
|
|
44
44
|
import { hostedBootCommand } from './hosted-boot.js';
|
|
45
45
|
import { mergeJobCommand } from './merge-job.js';
|
|
46
46
|
import { authedFetch } from './http.js';
|
|
@@ -48,7 +48,9 @@ import { resolveBaseUrl } from './http.js';
|
|
|
48
48
|
import { getWorkspaceRoot } from './workspace.js';
|
|
49
49
|
import { attachedWorkspaceId } from './workspace-sync.js';
|
|
50
50
|
import { tailWorkbenchRun } from './workbench-cli.js';
|
|
51
|
-
|
|
51
|
+
// Default hosted model — SINGLE source of truth in hosted-loop-lifecycle.ts
|
|
52
|
+
// (ALI-1086); re-exported here so existing importers keep working.
|
|
53
|
+
export { DEFAULT_HOSTED_MODEL };
|
|
52
54
|
export const DEFAULT_DURATION_MINUTES = 60;
|
|
53
55
|
export const MAX_DURATION_MINUTES = 24 * 60;
|
|
54
56
|
export const DEFAULT_ROTATION_INTERVAL_MS = 55 * 60 * 1000;
|
package/package.json
CHANGED
package/src/hosted-boot.ts
CHANGED
|
@@ -50,7 +50,7 @@ import {
|
|
|
50
50
|
type HostedLoopContext,
|
|
51
51
|
type HostedLoopResult,
|
|
52
52
|
} from './hosted-loop.js'
|
|
53
|
-
import { DEFAULT_EGRESS_CANARY_HOST } from './hosted-loop-lifecycle.js'
|
|
53
|
+
import { DEFAULT_EGRESS_CANARY_HOST, DEFAULT_HOSTED_MODEL } from './hosted-loop-lifecycle.js'
|
|
54
54
|
import { resumeRunEventSink } from './hosted-run-event-sink.js'
|
|
55
55
|
import { stageOrizuSkill } from './hosted-skill-staging.js'
|
|
56
56
|
|
|
@@ -62,8 +62,6 @@ export type BootFetch = (url: string, init?: RequestInit) => Promise<Response>
|
|
|
62
62
|
* DO-path `workers/session-coordinator/src/bootstrap.ts`); kept in sync by
|
|
63
63
|
* grep. Inlined (not imported) to avoid a hosted-session-cli import cycle. */
|
|
64
64
|
const ANTHROPIC_DUMMY_KEY = 'sk-ant-orizu-proxy-broker-placeholder'
|
|
65
|
-
/** Default hosted model — mirrors hosted-session-cli.ts `DEFAULT_HOSTED_MODEL`. */
|
|
66
|
-
const DEFAULT_HOSTED_MODEL = 'anthropic/claude-opus-4-8'
|
|
67
65
|
|
|
68
66
|
// -- Frozen env contract ------------------------------------------------------
|
|
69
67
|
|
|
@@ -513,6 +511,9 @@ export interface RunHostedBootOptions {
|
|
|
513
511
|
context: HostedLoopContext
|
|
514
512
|
taskPrompt: string
|
|
515
513
|
bearerProvider: () => string
|
|
514
|
+
/** Verbatim secrets the loop's event redaction must scrub (ALI-1062: the
|
|
515
|
+
* boot secret — bare hex, so no shape pattern would ever catch it). */
|
|
516
|
+
redactSecretsList: readonly string[]
|
|
516
517
|
}) => Promise<HostedLoopResult>
|
|
517
518
|
now?: () => number
|
|
518
519
|
log?: (line: string) => void
|
|
@@ -565,6 +566,17 @@ export async function runHostedBoot(opts: RunHostedBootOptions): Promise<HostedB
|
|
|
565
566
|
const log = opts.log ?? ((): void => {})
|
|
566
567
|
const now = opts.now ?? ((): number => Date.now())
|
|
567
568
|
|
|
569
|
+
// 0 — ENV HYGIENE (ALI-1062): the boot secret is the DO path's only durable
|
|
570
|
+
// credential (it mints agent bearers at the internet-reachable agent-token
|
|
571
|
+
// route for up to 24h), and `env.bootSecret` is already captured — so scrub
|
|
572
|
+
// the raw value from the process env NOW, before anything downstream can
|
|
573
|
+
// inherit it: the opencode/agent process is spawned with `{...process.env}`
|
|
574
|
+
// (nodeChildSpawner) and the customer `.orizu/setup.sh` hook inherits the
|
|
575
|
+
// env too. Nothing reads it from the env after this point — the rotation /
|
|
576
|
+
// boot-status closures hold `env.bootSecret`, and the git credential helper
|
|
577
|
+
// reads the 0600 run-dir boot-secret FILE (written in step 4).
|
|
578
|
+
delete processEnv.ORIZU_BOOT_SECRET
|
|
579
|
+
|
|
568
580
|
// 1 — Pull the agent bearer (retry/backoff — the DO may still be arming).
|
|
569
581
|
const bearer = await pullAgentBearer({
|
|
570
582
|
agentTokenUrl: env.agentTokenUrl,
|
|
@@ -738,10 +750,20 @@ export async function runHostedBoot(opts: RunHostedBootOptions): Promise<HostedB
|
|
|
738
750
|
const runLoop =
|
|
739
751
|
opts.runLoop ??
|
|
740
752
|
((input): Promise<HostedLoopResult> =>
|
|
741
|
-
runHostedLoop({
|
|
753
|
+
runHostedLoop({
|
|
754
|
+
context: input.context,
|
|
755
|
+
taskPrompt: input.taskPrompt,
|
|
756
|
+
bearerProvider: input.bearerProvider,
|
|
757
|
+
redactSecretsList: input.redactSecretsList,
|
|
758
|
+
}))
|
|
742
759
|
const result = await runLoop({
|
|
743
760
|
context: loopContext,
|
|
744
761
|
taskPrompt: session.task,
|
|
762
|
+
// ALI-1062 exact-value redaction: the boot secret is 64 bare hex chars —
|
|
763
|
+
// no TOKEN_SHAPE_PATTERNS rule can catch it, so it MUST ride the loop's
|
|
764
|
+
// verbatim redaction list or an echoed env dump would land it in
|
|
765
|
+
// workbench_run_events.
|
|
766
|
+
redactSecretsList: [env.bootSecret],
|
|
745
767
|
// Read the bearer FILE fresh per request (review F1): `startBearerRotation`
|
|
746
768
|
// rewrites `bearerFileAbs` before the current bearer expires, so a run
|
|
747
769
|
// exceeding the ~60-min TTL (DO cap is 24h) keeps a valid bearer. A frozen
|
|
@@ -155,6 +155,23 @@ const ANTHROPIC_ADAPTIVE_THINKING_MODELS = new Set([
|
|
|
155
155
|
])
|
|
156
156
|
const ANTHROPIC_ADAPTIVE_EFFORTS = new Set(['low', 'medium', 'high', 'xhigh', 'max'])
|
|
157
157
|
|
|
158
|
+
/**
|
|
159
|
+
* Split a possibly provider-qualified model string into provider + model id.
|
|
160
|
+
* SINGLE-SLASH ids ("anthropic/claude-opus-4-8") are the assumed form; a bare
|
|
161
|
+
* id defaults to provider "anthropic". This is THE shared parse — consumed by
|
|
162
|
+
* BOTH the prompt path (`buildPromptRequestBody`) and the pre-prompt
|
|
163
|
+
* validation (`awaitOpenCodeModelResolvable`), so the two can never drift:
|
|
164
|
+
* parity is exactly what makes the validation meaningful (ALI-1086).
|
|
165
|
+
* KNOWN LIMIT: `split('/', 2)` drops everything after the second slash, so a
|
|
166
|
+
* 3-part id ("openrouter/anthropic/claude-x") misparses to provider
|
|
167
|
+
* "openrouter" + model "anthropic" — identically in both places (today's
|
|
168
|
+
* pre-existing prompt-path behavior, now shared rather than duplicated).
|
|
169
|
+
*/
|
|
170
|
+
export function qualifyHostedModel(model: string): { providerId: string; modelId: string } {
|
|
171
|
+
const [providerId, modelId] = model.includes('/') ? model.split('/', 2) : ['anthropic', model]
|
|
172
|
+
return { providerId, modelId }
|
|
173
|
+
}
|
|
174
|
+
|
|
158
175
|
export function buildPromptRequestBody(
|
|
159
176
|
content: string,
|
|
160
177
|
model: string | undefined,
|
|
@@ -167,7 +184,9 @@ export function buildPromptRequestBody(
|
|
|
167
184
|
}
|
|
168
185
|
if (!model) return body
|
|
169
186
|
|
|
170
|
-
|
|
187
|
+
// Shared parse (see qualifyHostedModel): keeps the prompt path and the
|
|
188
|
+
// ALI-1086 pre-prompt validation in lockstep by construction.
|
|
189
|
+
const { providerId, modelId } = qualifyHostedModel(model)
|
|
171
190
|
const modelSpec: Record<string, unknown> = { providerID: providerId, modelID: modelId }
|
|
172
191
|
|
|
173
192
|
if (reasoningEffort) {
|
|
@@ -757,6 +776,190 @@ function errorEvent(messageId: string, error: unknown): HarnessEvent {
|
|
|
757
776
|
return { kind: 'error', messageId, critical: true, payload: { error: message } }
|
|
758
777
|
}
|
|
759
778
|
|
|
779
|
+
// -- Pre-prompt model validation (ALI-1086) -----------------------------------
|
|
780
|
+
//
|
|
781
|
+
// The pinned OpenCode (see OPENCODE_PINNED_VERSION) resolves model ids against
|
|
782
|
+
// a catalog that is its BUNDLED snapshot merged with a runtime fetch of
|
|
783
|
+
// https://models.dev/api.json (verified empirically on ALI-1086: fetched at
|
|
784
|
+
// boot when reachable, cached to <cache>/opencode/models.json, refreshed when
|
|
785
|
+
// stale). When models.dev is unreachable the bundled snapshot is stale
|
|
786
|
+
// (predates claude-opus-4-8) and the FIRST prompt dies inside the SSE stream
|
|
787
|
+
// with an opaque `run_failed: Model not found`. These helpers let the hosted
|
|
788
|
+
// loop ask the RUNNING `opencode serve` — `GET /config/providers` is the
|
|
789
|
+
// runtime's actual resolvable catalog — whether the requested model resolves,
|
|
790
|
+
// BEFORE the prompt is posted, and fail fast naming the alternatives.
|
|
791
|
+
|
|
792
|
+
/** provider id → model ids the RUNNING opencode instance can resolve. */
|
|
793
|
+
export type OpenCodeModelCatalog = ReadonlyMap<string, readonly string[]>
|
|
794
|
+
|
|
795
|
+
const MODEL_CATALOG_REQUEST_TIMEOUT_MS = 10_000
|
|
796
|
+
|
|
797
|
+
/**
|
|
798
|
+
* Read the resolvable catalog from a running `opencode serve` via
|
|
799
|
+
* `GET /config/providers` (shape: `{ providers: [{ id, models: { <id>: … } }] }`,
|
|
800
|
+
* verified against opencode-ai@1.14.41). Returns null on ANY transport/shape
|
|
801
|
+
* failure — the caller treats that as "cannot validate", never as "invalid".
|
|
802
|
+
*/
|
|
803
|
+
export async function fetchOpenCodeModelCatalog(
|
|
804
|
+
baseUrl: string,
|
|
805
|
+
fetchImpl: HarnessFetch = globalThis.fetch as HarnessFetch,
|
|
806
|
+
timeoutMs: number = MODEL_CATALOG_REQUEST_TIMEOUT_MS
|
|
807
|
+
): Promise<OpenCodeModelCatalog | null> {
|
|
808
|
+
const controller = new AbortController()
|
|
809
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
|
810
|
+
try {
|
|
811
|
+
const res = await fetchImpl(`${baseUrl.replace(/\/$/, '')}/config/providers`, {
|
|
812
|
+
signal: controller.signal,
|
|
813
|
+
})
|
|
814
|
+
if (!res.ok) return null
|
|
815
|
+
const data = asRecord(await res.json())
|
|
816
|
+
if (!Array.isArray(data.providers)) return null
|
|
817
|
+
const catalog = new Map<string, string[]>()
|
|
818
|
+
for (const entry of data.providers) {
|
|
819
|
+
const provider = asRecord(entry)
|
|
820
|
+
const id = typeof provider.id === 'string' ? provider.id : null
|
|
821
|
+
if (!id) continue
|
|
822
|
+
catalog.set(id, Object.keys(asRecord(provider.models)))
|
|
823
|
+
}
|
|
824
|
+
return catalog
|
|
825
|
+
} catch {
|
|
826
|
+
return null
|
|
827
|
+
} finally {
|
|
828
|
+
clearTimeout(timer)
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
function sharedPrefixLength(a: string, b: string): number {
|
|
833
|
+
const max = Math.min(a.length, b.length)
|
|
834
|
+
let i = 0
|
|
835
|
+
while (i < max && a[i] === b[i]) i += 1
|
|
836
|
+
return i
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/** Rank candidate model ids by similarity to the requested id (longest shared
|
|
840
|
+
* prefix first, ties broken lexicographically DESCENDING so the newest of a
|
|
841
|
+
* versioned family leads) and cap the list — the fail-fast error should lead
|
|
842
|
+
* with the closest/newest substitutes, not a full catalog dump. */
|
|
843
|
+
export function rankModelAlternatives(
|
|
844
|
+
requested: string,
|
|
845
|
+
candidates: readonly string[],
|
|
846
|
+
limit = 8
|
|
847
|
+
): string[] {
|
|
848
|
+
return [...new Set(candidates)]
|
|
849
|
+
.sort((a, b) => {
|
|
850
|
+
const byPrefix = sharedPrefixLength(requested, b) - sharedPrefixLength(requested, a)
|
|
851
|
+
return byPrefix !== 0 ? byPrefix : b.localeCompare(a)
|
|
852
|
+
})
|
|
853
|
+
.slice(0, limit)
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
export type ModelValidationOutcome =
|
|
857
|
+
| { kind: 'resolvable'; waitedMs: number }
|
|
858
|
+
| {
|
|
859
|
+
kind: 'unresolvable'
|
|
860
|
+
/** Provider-qualified requested model, e.g. "anthropic/claude-opus-4-8". */
|
|
861
|
+
requested: string
|
|
862
|
+
/** Provider-qualified resolvable substitutes, closest first. */
|
|
863
|
+
alternatives: string[]
|
|
864
|
+
/** False when the PROVIDER itself is unknown to the runtime. */
|
|
865
|
+
providerKnown: boolean
|
|
866
|
+
waitedMs: number
|
|
867
|
+
}
|
|
868
|
+
| { kind: 'skipped'; reason: string }
|
|
869
|
+
|
|
870
|
+
export interface AwaitModelResolvableOptions {
|
|
871
|
+
/** Base URL of the running `opencode serve`. */
|
|
872
|
+
baseUrl: string
|
|
873
|
+
/** Requested model, provider-qualified or bare (bare → "anthropic"). */
|
|
874
|
+
model: string
|
|
875
|
+
fetchImpl?: HarnessFetch
|
|
876
|
+
/**
|
|
877
|
+
* Total budget to wait for the model to become resolvable. OpenCode's
|
|
878
|
+
* models.dev refresh runs in the BACKGROUND at boot, so the catalog can be
|
|
879
|
+
* bundled-only for the first seconds after the server answers HTTP; polling
|
|
880
|
+
* bridges that window instead of failing a run the refresh would have fixed.
|
|
881
|
+
*/
|
|
882
|
+
timeoutMs?: number
|
|
883
|
+
pollMs?: number
|
|
884
|
+
sleep?: (ms: number) => Promise<void>
|
|
885
|
+
now?: () => number
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
const MODEL_RESOLVE_TIMEOUT_MS = 30_000
|
|
889
|
+
const MODEL_RESOLVE_POLL_MS = 2_000
|
|
890
|
+
|
|
891
|
+
/**
|
|
892
|
+
* Decide whether the requested model is resolvable by the RUNNING opencode,
|
|
893
|
+
* polling briefly so the boot-time models.dev refresh has time to land.
|
|
894
|
+
* Fail-open on validator infrastructure, with a DELIBERATE asymmetry on null
|
|
895
|
+
* catalog reads:
|
|
896
|
+
* - null BEFORE any successful read → `skipped` immediately (dead/absent
|
|
897
|
+
* endpoint; opencode's own `session.error` remains the backstop, and the
|
|
898
|
+
* early exit keeps absent-endpoint runs — and tests — cheap);
|
|
899
|
+
* - null AFTER at least one successful read → a transient boot-window blip
|
|
900
|
+
* (5xx/drop): sleep and keep polling within the budget; a budget that
|
|
901
|
+
* ends in that state returns `skipped`, NEVER a false `unresolvable` off
|
|
902
|
+
* a flaky read.
|
|
903
|
+
* Only a HEALTHY catalog that never lists the model within the budget yields
|
|
904
|
+
* `unresolvable`, with ranked, provider-qualified alternatives.
|
|
905
|
+
*/
|
|
906
|
+
export async function awaitOpenCodeModelResolvable(
|
|
907
|
+
opts: AwaitModelResolvableOptions
|
|
908
|
+
): Promise<ModelValidationOutcome> {
|
|
909
|
+
const timeoutMs = opts.timeoutMs ?? MODEL_RESOLVE_TIMEOUT_MS
|
|
910
|
+
const pollMs = opts.pollMs ?? MODEL_RESOLVE_POLL_MS
|
|
911
|
+
const sleep = opts.sleep ?? ((ms: number): Promise<void> => new Promise(r => setTimeout(r, ms)))
|
|
912
|
+
const now = opts.now ?? ((): number => Date.now())
|
|
913
|
+
const { providerId, modelId } = qualifyHostedModel(opts.model)
|
|
914
|
+
const requested = `${providerId}/${modelId}`
|
|
915
|
+
|
|
916
|
+
const startedAt = now()
|
|
917
|
+
let lastGoodCatalog: OpenCodeModelCatalog | null = null
|
|
918
|
+
let sawTransientFailure = false
|
|
919
|
+
for (;;) {
|
|
920
|
+
const catalog = await fetchOpenCodeModelCatalog(opts.baseUrl, opts.fetchImpl)
|
|
921
|
+
if (catalog === null) {
|
|
922
|
+
if (lastGoodCatalog === null) {
|
|
923
|
+
// Never answered — the endpoint is dead/absent, not blipping. Skip
|
|
924
|
+
// immediately rather than invent a failure mode (or a slow wait) the
|
|
925
|
+
// run would not otherwise have.
|
|
926
|
+
return { kind: 'skipped', reason: 'opencode /config/providers unavailable' }
|
|
927
|
+
}
|
|
928
|
+
// Transient failure after a successful read (boot-window 5xx/drop) —
|
|
929
|
+
// keep polling within the budget.
|
|
930
|
+
sawTransientFailure = true
|
|
931
|
+
} else {
|
|
932
|
+
lastGoodCatalog = catalog
|
|
933
|
+
sawTransientFailure = false
|
|
934
|
+
if ((catalog.get(providerId) ?? []).includes(modelId)) {
|
|
935
|
+
return { kind: 'resolvable', waitedMs: now() - startedAt }
|
|
936
|
+
}
|
|
937
|
+
// Deliberately NOT failing fast when the provider itself is unknown:
|
|
938
|
+
// the models.dev refresh can add whole providers too, so the poll runs
|
|
939
|
+
// the full budget either way.
|
|
940
|
+
}
|
|
941
|
+
if (now() - startedAt >= timeoutMs) break
|
|
942
|
+
await sleep(pollMs)
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
if (sawTransientFailure || lastGoodCatalog === null) {
|
|
946
|
+
// The budget ended on a flaky read — never declare `unresolvable` off it.
|
|
947
|
+
return { kind: 'skipped', reason: 'opencode /config/providers became unavailable mid-poll' }
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
const providerKnown = lastGoodCatalog.has(providerId)
|
|
951
|
+
const candidates = providerKnown
|
|
952
|
+
? (lastGoodCatalog.get(providerId) ?? []).map(id => `${providerId}/${id}`)
|
|
953
|
+
: [...lastGoodCatalog.entries()].flatMap(([pid, ids]) => ids.map(id => `${pid}/${id}`))
|
|
954
|
+
return {
|
|
955
|
+
kind: 'unresolvable',
|
|
956
|
+
requested,
|
|
957
|
+
alternatives: rankModelAlternatives(requested, candidates),
|
|
958
|
+
providerKnown,
|
|
959
|
+
waitedMs: now() - startedAt,
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
|
|
760
963
|
// -- spawnOpenCode: the P3.5 in-sandbox launch helper ------------------------
|
|
761
964
|
|
|
762
965
|
export interface OpenCodeConfigOptions {
|
|
@@ -24,6 +24,27 @@
|
|
|
24
24
|
|
|
25
25
|
import type { HarnessEvent } from './hosted-harness.js'
|
|
26
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Default provider-qualified hosted model — the SINGLE source of truth
|
|
29
|
+
* (ALI-1086). Imported by both the operator-path session CLI
|
|
30
|
+
* (`hosted-session-cli.ts`) and the DO-path boot (`hosted-boot.ts`); never
|
|
31
|
+
* re-inline it.
|
|
32
|
+
*
|
|
33
|
+
* PIN CONSTRAINT: the hosted OpenCode runtime is HARD-PINNED to
|
|
34
|
+
* `opencode-ai@1.14.41` (`OPENCODE_PINNED_VERSION`, SSE-fragile — do not bump)
|
|
35
|
+
* whose BUNDLED model catalog predates this model (newest bundled anthropic
|
|
36
|
+
* opus: claude-opus-4-7). Verified empirically on ALI-1086: 1.14.41 DOES fetch
|
|
37
|
+
* `https://models.dev/api.json` at boot when reachable (cached to
|
|
38
|
+
* `~/.cache/opencode/models.json`, refreshed when stale, disabled only by
|
|
39
|
+
* `OPENCODE_DISABLE_MODELS_FETCH`), so with models.dev on the sandbox egress
|
|
40
|
+
* allowlist (#1392, `ORIZU_HOSTED_EGRESS_ALLOWLIST`) this id resolves in new
|
|
41
|
+
* sandboxes. If the catalog fetch is blocked/stale anyway, the loop's
|
|
42
|
+
* pre-prompt validation (`awaitOpenCodeModelResolvable`) fails the run fast,
|
|
43
|
+
* naming the resolvable alternatives. The durable fix for the stale bundled
|
|
44
|
+
* catalog is the ALI-929 harness swap.
|
|
45
|
+
*/
|
|
46
|
+
export const DEFAULT_HOSTED_MODEL = 'anthropic/claude-opus-4-8'
|
|
47
|
+
|
|
27
48
|
/**
|
|
28
49
|
* The run-scoped context the host orchestrator writes for the loop. Paths are
|
|
29
50
|
* ABSOLUTE (or sandbox-root-relative) so the loop resolves them regardless of
|
package/src/hosted-loop.ts
CHANGED
|
@@ -51,8 +51,10 @@ import { annotateHeadlessQuestions, withIdleWatchdog } from './hosted-headless.j
|
|
|
51
51
|
import {
|
|
52
52
|
DEFAULT_PROMPT_MAX_DURATION_MS,
|
|
53
53
|
OPENCODE_PINNED_VERSION,
|
|
54
|
+
awaitOpenCodeModelResolvable,
|
|
54
55
|
createOpenCodeHarness,
|
|
55
56
|
spawnOpenCode,
|
|
57
|
+
type ModelValidationOutcome,
|
|
56
58
|
type SpawnOpenCodeOptions,
|
|
57
59
|
type SpawnedOpenCode,
|
|
58
60
|
} from './hosted-harness-opencode.js'
|
|
@@ -60,6 +62,8 @@ import { createClaudeAgentHarness } from './hosted-harness-claude.js'
|
|
|
60
62
|
import {
|
|
61
63
|
drainHarnessToSink,
|
|
62
64
|
resumeRunEventSink,
|
|
65
|
+
RunTerminalError,
|
|
66
|
+
TerminalDeliveryError,
|
|
63
67
|
type HostedFetch,
|
|
64
68
|
type TerminalStatus,
|
|
65
69
|
} from './hosted-run-event-sink.js'
|
|
@@ -80,6 +84,7 @@ import {
|
|
|
80
84
|
// coordinator. Re-exported here so this module's public surface is unchanged.
|
|
81
85
|
export {
|
|
82
86
|
DEFAULT_EGRESS_CANARY_HOST,
|
|
87
|
+
DEFAULT_HOSTED_MODEL,
|
|
83
88
|
buildOpenCodeSpawnEnv,
|
|
84
89
|
egressCanaryAllowedHost,
|
|
85
90
|
runEgressCanary,
|
|
@@ -140,6 +145,16 @@ export interface RunHostedLoopOptions {
|
|
|
140
145
|
createClaudeHarness?: () => AgentHarness
|
|
141
146
|
spawnOpenCode?: (opts: SpawnOpenCodeOptions) => SpawnedOpenCode | Promise<SpawnedOpenCode>
|
|
142
147
|
installOpenCode?: () => Promise<InstallResult>
|
|
148
|
+
/**
|
|
149
|
+
* Pre-prompt model validation against the RUNNING opencode's resolvable
|
|
150
|
+
* catalog (ALI-1086). Default: `awaitOpenCodeModelResolvable`, which polls
|
|
151
|
+
* `GET /config/providers` briefly so the boot-time models.dev refresh can
|
|
152
|
+
* land. An `unresolvable` outcome fails the run BEFORE the first prompt with
|
|
153
|
+
* an error naming the requested id and the resolvable alternatives; a
|
|
154
|
+
* `skipped` outcome (catalog endpoint unreachable) proceeds — opencode's own
|
|
155
|
+
* `session.error` remains the backstop. Injectable for tests.
|
|
156
|
+
*/
|
|
157
|
+
validateModel?: (input: { baseUrl: string; model: string }) => Promise<ModelValidationOutcome>
|
|
143
158
|
now?: () => number
|
|
144
159
|
signal?: AbortSignal
|
|
145
160
|
/** Extra verbatim secrets to redact (the bearer is added by the sink itself). */
|
|
@@ -170,6 +185,9 @@ export interface RunHostedLoopOptions {
|
|
|
170
185
|
* `ORIZU_AGENT_IDLE_TIMEOUT_MS` or 25 minutes. <= 0 disables the watchdog.
|
|
171
186
|
*/
|
|
172
187
|
idleTimeoutMs?: number
|
|
188
|
+
/** Injectable sleep for the terminal-delivery retry backoff (ALI-1065
|
|
189
|
+
* finding 2). Tests pass a no-op to avoid real delays. */
|
|
190
|
+
sleepImpl?: (ms: number) => Promise<void>
|
|
173
191
|
}
|
|
174
192
|
|
|
175
193
|
export interface HostedLoopResult {
|
|
@@ -211,6 +229,16 @@ function resolveIdleTimeoutMs(override: number | undefined): number {
|
|
|
211
229
|
return DEFAULT_AGENT_IDLE_TIMEOUT_MS
|
|
212
230
|
}
|
|
213
231
|
|
|
232
|
+
/**
|
|
233
|
+
* Loop-level retry schedule for an undeliverable TERMINAL transition (ALI-1065
|
|
234
|
+
* finding 2). Each entry is a pause before one more full sink delivery cycle
|
|
235
|
+
* (which itself retries ~4 times over ~350ms), so a transport blip of a few
|
|
236
|
+
* seconds recovers with the ORIGINAL status instead of being re-recorded as
|
|
237
|
+
* 'failed'. Bounded: a persistently unreachable control plane fails the
|
|
238
|
+
* DELIVERY visibly (result.error) without recording a wrong outcome.
|
|
239
|
+
*/
|
|
240
|
+
export const TERMINAL_DELIVERY_RETRY_DELAYS_MS: readonly number[] = [2_000, 8_000]
|
|
241
|
+
|
|
214
242
|
/** Model-key env vars whose value must be scrubbed from every run event. */
|
|
215
243
|
const MODEL_KEY_ENV_VARS = ['ANTHROPIC_API_KEY', 'OPENAI_API_KEY']
|
|
216
244
|
|
|
@@ -515,6 +543,43 @@ export async function runHostedLoop(opts: RunHostedLoopOptions): Promise<HostedL
|
|
|
515
543
|
})
|
|
516
544
|
}
|
|
517
545
|
|
|
546
|
+
// Fail-fast model validation (ALI-1086): ask the RUNNING opencode whether
|
|
547
|
+
// the requested model is resolvable BEFORE the first prompt. The pinned
|
|
548
|
+
// opencode's bundled catalog is stale (predates claude-opus-4-8) and only
|
|
549
|
+
// a boot-time models.dev fetch (#1392 allowlists it) refreshes it — when
|
|
550
|
+
// that fetch is blocked, the first prompt used to die inside the SSE
|
|
551
|
+
// stream with an opaque `run_failed: Model not found`. Now the run fails
|
|
552
|
+
// immediately with a structured artifact + an error naming the requested
|
|
553
|
+
// id AND the resolvable alternatives. Fail-open on validator
|
|
554
|
+
// infrastructure: an unreachable catalog endpoint (`skipped`) proceeds
|
|
555
|
+
// silently — opencode's own session.error remains the backstop. Healthy
|
|
556
|
+
// and skipped runs append NO event, so the ALI-929 cross-harness
|
|
557
|
+
// stream-identity invariant is untouched (validation exists only on the
|
|
558
|
+
// opencode path; the Claude-SDK driver resolves models itself).
|
|
559
|
+
const validateModel = opts.validateModel ?? awaitOpenCodeModelResolvable
|
|
560
|
+
const validation = await validateModel({ baseUrl: spawned.baseUrl, model: context.model })
|
|
561
|
+
if (validation.kind === 'unresolvable') {
|
|
562
|
+
await sink.append({
|
|
563
|
+
kind: 'artifact',
|
|
564
|
+
payload: {
|
|
565
|
+
step: 'model_validation',
|
|
566
|
+
ok: false,
|
|
567
|
+
requestedModel: validation.requested,
|
|
568
|
+
resolvableAlternatives: validation.alternatives,
|
|
569
|
+
providerKnown: validation.providerKnown,
|
|
570
|
+
opencodeVersion: pinnedVersion,
|
|
571
|
+
waitedMs: validation.waitedMs,
|
|
572
|
+
},
|
|
573
|
+
})
|
|
574
|
+
throw new Error(
|
|
575
|
+
`requested model ${validation.requested} is not resolvable by the pinned ` +
|
|
576
|
+
`opencode runtime (opencode-ai@${pinnedVersion}); resolvable alternatives: ` +
|
|
577
|
+
`${validation.alternatives.join(', ') || '(none)'} — retry with --model ` +
|
|
578
|
+
`set to one of these (stale bundled catalog + blocked models.dev refresh; ` +
|
|
579
|
+
`see ALI-1086, durable fix ALI-929)`
|
|
580
|
+
)
|
|
581
|
+
}
|
|
582
|
+
|
|
518
583
|
// Derive the per-prompt max-duration cap from the sandbox budget so a long
|
|
519
584
|
// `--duration` run is not killed at the hard-coded 90-min cap while its
|
|
520
585
|
// sandbox is still alive; floored at the harness default for short/undated
|
|
@@ -591,10 +656,58 @@ export async function runHostedLoop(opts: RunHostedLoopOptions): Promise<HostedL
|
|
|
591
656
|
return { status, agentSessionId, installOk, error: null }
|
|
592
657
|
} catch (error) {
|
|
593
658
|
const message = error instanceof Error ? error.message : String(error)
|
|
594
|
-
//
|
|
595
|
-
//
|
|
596
|
-
// is valuable, incl. an idle-watchdog abort) — before the terminal seals.
|
|
659
|
+
// Harvest FIRST on every abnormal path (partial work is valuable, incl. an
|
|
660
|
+
// idle-watchdog abort) — before any terminal write seals the sink.
|
|
597
661
|
await runAutoHarvest()
|
|
662
|
+
|
|
663
|
+
// ALI-1065 finding 2: an undeliverable TERMINAL transition must NEVER be
|
|
664
|
+
// re-recorded as 'failed' — the run's outcome was already decided; only its
|
|
665
|
+
// DELIVERY failed. Retry the delivery with the ORIGINAL status (spaced past
|
|
666
|
+
// the sink's own ~350ms retry window, so a ~1s transport blip recovers),
|
|
667
|
+
// and if it still cannot land, fail the DELIVERY visibly: return the
|
|
668
|
+
// intended status with a non-null error, which the boot reports via its
|
|
669
|
+
// boot-status callback, and the coordinator's session-end finalizer sweeps
|
|
670
|
+
// the still-'running' row server-side with an explicit marker.
|
|
671
|
+
if (error instanceof TerminalDeliveryError) {
|
|
672
|
+
const sleep = opts.sleepImpl ?? ((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)))
|
|
673
|
+
for (const delayMs of TERMINAL_DELIVERY_RETRY_DELAYS_MS) {
|
|
674
|
+
if (sink.sealed) break
|
|
675
|
+
await sleep(delayMs)
|
|
676
|
+
try {
|
|
677
|
+
await sink.finish(error.intendedStatus, error.finishOptions)
|
|
678
|
+
return { status: error.intendedStatus, agentSessionId, installOk, error: null }
|
|
679
|
+
} catch (retryError) {
|
|
680
|
+
// The run went terminal server-side while we were retrying: accept
|
|
681
|
+
// the server record (Orizu records win) instead of retrying further.
|
|
682
|
+
if (retryError instanceof RunTerminalError) {
|
|
683
|
+
return { status: 'cancelled', agentSessionId, installOk, error: null }
|
|
684
|
+
}
|
|
685
|
+
// Otherwise keep retrying within the bounded schedule; never change
|
|
686
|
+
// the status.
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
if (sink.sealed) {
|
|
690
|
+
// A concurrent/preceding delivery landed after all — the record is
|
|
691
|
+
// terminal server-side with our intended status semantics preserved.
|
|
692
|
+
return { status: error.intendedStatus, agentSessionId, installOk, error: null }
|
|
693
|
+
}
|
|
694
|
+
return {
|
|
695
|
+
status: error.intendedStatus,
|
|
696
|
+
agentSessionId,
|
|
697
|
+
installOk,
|
|
698
|
+
error: `terminal delivery failed (intended status preserved): ${message}`,
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
// The run went terminal SERVER-side (cancel/finish out from under this
|
|
703
|
+
// writer): the sink sealed itself; accept the server record (Orizu records
|
|
704
|
+
// win) instead of fabricating a local failure.
|
|
705
|
+
if (error instanceof RunTerminalError) {
|
|
706
|
+
return { status: 'cancelled', agentSessionId, installOk, error: null }
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
// Fail closed: never leave the run non-terminal. finish() is idempotent and
|
|
710
|
+
// no-ops when the drain already sealed the sink.
|
|
598
711
|
if (!sink.sealed) {
|
|
599
712
|
try {
|
|
600
713
|
await sink.finish('failed', { summary: { error: message } })
|
|
@@ -161,6 +161,27 @@ export class CriticalDeliveryError extends Error {
|
|
|
161
161
|
}
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Raised when `finish()` itself is undeliverable (ALI-1065 finding 2): a
|
|
166
|
+
* CriticalDeliveryError thrown WHILE DELIVERING A TERMINAL TRANSITION, carrying
|
|
167
|
+
* the intended status + finish options so the caller can retry the delivery
|
|
168
|
+
* with the ORIGINAL outcome. Before this, the error surfaced as a bare
|
|
169
|
+
* CriticalDeliveryError and the loop's generic catch re-finished the run
|
|
170
|
+
* 'failed' — a ~1s transport blip on the SUCCESS terminal recorded a succeeded
|
|
171
|
+
* run as failed. Subclasses CriticalDeliveryError so existing classification
|
|
172
|
+
* (drop-vs-surface) is unchanged.
|
|
173
|
+
*/
|
|
174
|
+
export class TerminalDeliveryError extends CriticalDeliveryError {
|
|
175
|
+
readonly intendedStatus: TerminalStatus
|
|
176
|
+
readonly finishOptions: FinishOptions | undefined
|
|
177
|
+
constructor(message: string, intendedStatus: TerminalStatus, finishOptions?: FinishOptions) {
|
|
178
|
+
super(message)
|
|
179
|
+
this.name = 'TerminalDeliveryError'
|
|
180
|
+
this.intendedStatus = intendedStatus
|
|
181
|
+
this.finishOptions = finishOptions
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
164
185
|
/**
|
|
165
186
|
* Raised on a 403 from the RunAPI. Named for what a 403 actually is — NOT "the
|
|
166
187
|
* run is terminal" (that is 404/410). A 403 means the caller may no longer write
|
|
@@ -534,17 +555,31 @@ export function createRunEventSink(options: CreateRunEventSinkOptions): RunEvent
|
|
|
534
555
|
diag(`finish(${status}) called on an already-sealed sink — ignored (idempotent)`)
|
|
535
556
|
return
|
|
536
557
|
}
|
|
537
|
-
//
|
|
538
|
-
//
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
)
|
|
558
|
+
// ALI-1065 finding 2: a delivery failure ANYWHERE inside finish (the
|
|
559
|
+
// transcript tail is a critical append; the terminal PATCH is the
|
|
560
|
+
// transition itself) must carry the INTENDED status out to the caller.
|
|
561
|
+
// Without this, the loop's generic catch converted an undeliverable
|
|
562
|
+
// SUCCESS terminal into a recorded 'failed'. RunTerminalError/RunAuthError
|
|
563
|
+
// pass through unchanged — they are classifications, not delivery
|
|
564
|
+
// failures.
|
|
565
|
+
try {
|
|
566
|
+
// Pre-terminal appends (still writable): flush coalesced tokens, then the
|
|
567
|
+
// bounded transcript tail (the seal below refuses any later append).
|
|
568
|
+
await flushTokens()
|
|
569
|
+
if (opts?.transcript) {
|
|
570
|
+
await postEvent(
|
|
571
|
+
AGENT_TRANSCRIPT_EVENT_TYPE,
|
|
572
|
+
{ text: tail(opts.transcript, TRANSCRIPT_TAIL_MAX_CHARS), truncated: opts.transcript.length > TRANSCRIPT_TAIL_MAX_CHARS },
|
|
573
|
+
true
|
|
574
|
+
)
|
|
575
|
+
}
|
|
576
|
+
await patchTerminal(status, opts)
|
|
577
|
+
} catch (error) {
|
|
578
|
+
if (error instanceof CriticalDeliveryError && !(error instanceof TerminalDeliveryError)) {
|
|
579
|
+
throw new TerminalDeliveryError(error.message, status, opts)
|
|
580
|
+
}
|
|
581
|
+
throw error
|
|
546
582
|
}
|
|
547
|
-
await patchTerminal(status, opts)
|
|
548
583
|
}
|
|
549
584
|
|
|
550
585
|
return {
|
|
@@ -661,6 +696,39 @@ export async function drainHarnessToSink(
|
|
|
661
696
|
beforeFinishRan = true
|
|
662
697
|
if (opts.beforeFinish) await opts.beforeFinish()
|
|
663
698
|
}
|
|
699
|
+
// ALI-1065 finding 3: the server now rejects appends to a terminal run with
|
|
700
|
+
// 410 (the run was cancelled/finished out from under this writer). The sink
|
|
701
|
+
// seals itself and raises RunTerminalError; the drain must treat that as a
|
|
702
|
+
// clean server-side termination — stop consuming and report 'cancelled' —
|
|
703
|
+
// not crash the loop into a spurious re-finish (Orizu records win; the
|
|
704
|
+
// stored status is whatever the server committed). The hook still runs so
|
|
705
|
+
// partial work is harvested, but with a sealed sink its own appends no-op.
|
|
706
|
+
const handleServerTerminal = async (): Promise<TerminalStatus> => {
|
|
707
|
+
await runBeforeFinish()
|
|
708
|
+
return 'cancelled'
|
|
709
|
+
}
|
|
710
|
+
// finish() can ALSO surface RunTerminalError (its pre-terminal token flush /
|
|
711
|
+
// transcript append hits the 410 before the PATCH ever runs) — accept the
|
|
712
|
+
// server's termination the same way instead of crashing the drain. A finish
|
|
713
|
+
// whose PATCH meets an already-terminal run does NOT throw (patchTerminal
|
|
714
|
+
// seals on a terminal 409), so this wrapper changes nothing on that path.
|
|
715
|
+
//
|
|
716
|
+
// SEALED-BEFORE-FINISH: the beforeFinish hook's own best-effort appends
|
|
717
|
+
// (auto-harvest events) swallow errors — including the RunTerminalError of a
|
|
718
|
+
// 410 that SEALED the sink. finish() on a sealed sink is an idempotent no-op,
|
|
719
|
+
// so without this check the drain would report the intended status as
|
|
720
|
+
// delivered when in fact the server terminated the run out from under us.
|
|
721
|
+
const finishAccepting = async (status: TerminalStatus, finishOpts: FinishOptions): Promise<TerminalStatus> => {
|
|
722
|
+
await runBeforeFinish()
|
|
723
|
+
if (sink.sealed) return 'cancelled'
|
|
724
|
+
try {
|
|
725
|
+
await sink.finish(status, finishOpts)
|
|
726
|
+
return status
|
|
727
|
+
} catch (error) {
|
|
728
|
+
if (error instanceof RunTerminalError) return 'cancelled'
|
|
729
|
+
throw error
|
|
730
|
+
}
|
|
731
|
+
}
|
|
664
732
|
for await (const event of events) {
|
|
665
733
|
if (event.kind === 'execution_complete') {
|
|
666
734
|
// A prompt the caller/agent stopped is a cancellation, not a failure: the
|
|
@@ -668,40 +736,39 @@ export async function drainHarnessToSink(
|
|
|
668
736
|
// and stop()-initiated paths. A genuine failure (success:false, no abort
|
|
669
737
|
// flag) still lands 'failed'.
|
|
670
738
|
if (event.payload.aborted === true) {
|
|
671
|
-
|
|
672
|
-
await sink.finish('cancelled', {
|
|
739
|
+
return finishAccepting('cancelled', {
|
|
673
740
|
summary: opts.summary,
|
|
674
741
|
evidence: opts.evidence,
|
|
675
742
|
transcript: opts.transcript,
|
|
676
743
|
})
|
|
677
|
-
return 'cancelled'
|
|
678
744
|
}
|
|
679
745
|
const success = event.payload.success !== false
|
|
680
746
|
const status: TerminalStatus = success ? 'succeeded' : 'failed'
|
|
681
747
|
const summary = success
|
|
682
748
|
? opts.summary
|
|
683
749
|
: { ...(opts.summary ?? {}), error: event.payload.error ?? 'execution failed' }
|
|
684
|
-
|
|
685
|
-
await sink.finish(status, { summary, evidence: opts.evidence, transcript: opts.transcript })
|
|
686
|
-
return status
|
|
750
|
+
return finishAccepting(status, { summary, evidence: opts.evidence, transcript: opts.transcript })
|
|
687
751
|
}
|
|
688
752
|
if (event.kind === 'error') {
|
|
689
|
-
|
|
690
|
-
await sink.finish('failed', {
|
|
753
|
+
return finishAccepting('failed', {
|
|
691
754
|
summary: { ...(opts.summary ?? {}), error: event.payload.error ?? 'agent error' },
|
|
692
755
|
evidence: opts.evidence,
|
|
693
756
|
transcript: opts.transcript,
|
|
694
757
|
})
|
|
695
|
-
return 'failed'
|
|
696
758
|
}
|
|
697
|
-
|
|
759
|
+
try {
|
|
760
|
+
await sink.append(event)
|
|
761
|
+
} catch (error) {
|
|
762
|
+
if (error instanceof RunTerminalError) return handleServerTerminal()
|
|
763
|
+
throw error
|
|
764
|
+
}
|
|
698
765
|
}
|
|
699
766
|
// The harness contract guarantees a terminal event; reaching here means the
|
|
700
767
|
// stream ended abnormally. Fail closed so the run never dangles non-terminal.
|
|
701
768
|
await runBeforeFinish()
|
|
702
|
-
|
|
769
|
+
if (sink.sealed) return 'cancelled'
|
|
770
|
+
return finishAccepting('failed', {
|
|
703
771
|
summary: { ...(opts.summary ?? {}), error: 'harness stream ended without a terminal event' },
|
|
704
772
|
transcript: opts.transcript,
|
|
705
773
|
})
|
|
706
|
-
return 'failed'
|
|
707
774
|
}
|
|
@@ -52,7 +52,7 @@ import { resumeRunEventSink } from './hosted-run-event-sink.js'
|
|
|
52
52
|
import { AGENT_GIT_IDENTITY, BEARER_BASENAME } from './hosted-runtime-assets.js'
|
|
53
53
|
import { createVercelProvider } from './vercel-sandbox-provider.js'
|
|
54
54
|
import { buildEgressPolicy, DEFAULT_EGRESS_CANARY_HOST } from './egress-policy.js'
|
|
55
|
-
import { hostedLoopCommand, type HostedLoopContext } from './hosted-loop.js'
|
|
55
|
+
import { DEFAULT_HOSTED_MODEL, hostedLoopCommand, type HostedLoopContext } from './hosted-loop.js'
|
|
56
56
|
import { hostedBootCommand } from './hosted-boot.js'
|
|
57
57
|
import { mergeJobCommand } from './merge-job.js'
|
|
58
58
|
import { authedFetch } from './http.js'
|
|
@@ -61,7 +61,9 @@ import { getWorkspaceRoot } from './workspace.js'
|
|
|
61
61
|
import { attachedWorkspaceId } from './workspace-sync.js'
|
|
62
62
|
import { tailWorkbenchRun } from './workbench-cli.js'
|
|
63
63
|
|
|
64
|
-
|
|
64
|
+
// Default hosted model — SINGLE source of truth in hosted-loop-lifecycle.ts
|
|
65
|
+
// (ALI-1086); re-exported here so existing importers keep working.
|
|
66
|
+
export { DEFAULT_HOSTED_MODEL }
|
|
65
67
|
export const DEFAULT_DURATION_MINUTES = 60
|
|
66
68
|
export const MAX_DURATION_MINUTES = 24 * 60
|
|
67
69
|
export const DEFAULT_ROTATION_INTERVAL_MS = 55 * 60 * 1000
|