mixdog 1.4.1 → 1.4.2

package/src/runtime/agent/orchestrator/providers/model-cache.mjs

@@ -1,7 +1,31 @@
 import { readFileSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { getPluginData } from '../config.mjs';
-import { updateJsonAtomicSync, writeJsonAtomicSync } from '../../../shared/atomic-file.mjs';
+import { updateJsonAtomic, writeJsonAtomicAsync } from '../../../shared/atomic-file.mjs';
+
+// Disk writes never block the event loop: they run on the async atomic-file
+// primitives, one at a time per cache file, in the order they were requested
+// (a later write is never overtaken by an earlier one finishing late). Every
+// instance of one file shares these module-level tables, since several
+// instances and several providers can address the same path.
+const writeTails = new Map(); // path → tail of that file's write chain
+// path → JSON text of the newest save() whose write has not finished. Sync
+// readers see it, so a save is visible at once although its disk write is not.
+const unsettledSaves = new Map();
+
+function enqueueWrite(p, task) {
+  const run = (writeTails.get(p) ?? Promise.resolve()).then(task);
+  writeTails.set(p, run);
+  run.then(() => {
+    if (writeTails.get(p) === run) writeTails.delete(p);
+  });
+  return run;
+}
+
+/** Resolves once every cache write requested so far has settled (shutdown, tests). */
+export async function flushModelCacheWrites() {
+  while (writeTails.size) await Promise.all([...writeTails.values()]);
+}
 
 // Shared model-catalog disk-cache CRUD, parameterized over the small bits each
 // provider varies: cache file name, TTL, an optional schema-version gate, and an
@@ -15,9 +39,10 @@ export function makeModelCache({ fileName, ttlMs, version = null, onSave = null
   // allowStale skips the TTL only; the schema-version gate still applies.
   function loadEntrySync({ allowStale = false } = {}) {
     const p = path();
-    if (!existsSync(p)) return null;
+    const unsettled = unsettledSaves.get(p);
+    if (unsettled === undefined && !existsSync(p)) return null;
     try {
-      const raw = JSON.parse(readFileSync(p, 'utf-8'));
+      const raw = JSON.parse(unsettled ?? readFileSync(p, 'utf-8'));
       if (version != null && raw?.version !== version) return null;
       if (!raw?.fetchedAt || !Array.isArray(raw.models)) return null;
       if (!allowStale && Date.now() - raw.fetchedAt > ttlMs) return null;
@@ -31,56 +56,98 @@ export function makeModelCache({ fileName, ttlMs, version = null, onSave = null
     return loadEntrySync(options)?.models ?? null;
   }
 
+  // The in-memory state (sync readers, onSave mirror) updates at once; the
+  // returned promise settles, never rejecting, when the disk write has.
   function save(models, extra = {}) {
+    const p = path();
+    // The value as of this call: later mutation of `models` must not change
+    // what lands on disk (the old synchronous write captured it the same way).
+    let text;
+    let record;
+    try {
+      text = JSON.stringify({
+        ...extra,
+        ...(version != null ? { version } : {}),
+        fetchedAt: Date.now(),
+        models,
+      });
+      record = JSON.parse(text);
+      unsettledSaves.set(p, text);
+    } catch {
+      return Promise.resolve();
+    }
     try {
-      writeJsonAtomicSync(
-        path(),
-        {
-          ...extra,
-          ...(version != null ? { version } : {}),
-          fetchedAt: Date.now(),
-          models,
-        },
-        { lock: true, fsyncDir: true }
-      );
       if (onSave) onSave(models);
     } catch {
       /* best-effort */
     }
+    return enqueueWrite(p, async () => {
+      try {
+        await writeJsonAtomicAsync(p, record, { lock: true, fsyncDir: true });
+      } catch {
+        /* best-effort */
+      } finally {
+        if (unsettledSaves.get(p) === text) unsettledSaves.delete(p);
+      }
+    });
   }
 
   // A discovery request claims a generation when it starts. Its roster is
   // written only if no newer request started meanwhile — in this process
   // (generation) or another one (`requestedAt` checked under the file lock) —
   // and `isCurrent()` (the credential it was fetched with is still active) holds.
+  // The returned publish() resolves, never rejecting, to whether the roster was
+  // written; the cross-process check runs under the file lock, so the answer
+  // is only known once the queued write has run.
   let generation = 0;
   function beginSave(isCurrent) {
     const claimed = ++generation;
     const requestedAt = Date.now();
+    const stillWanted = () => claimed === generation && isCurrent();
     return (models, extra = {}) => {
-      if (claimed !== generation || !isCurrent()) return false;
-      let written = false;
+      if (!stillWanted()) return Promise.resolve(false);
+      const p = path();
+      // The roster as of this call, not as mutated while the write waits.
+      let snapshot;
       try {
-        updateJsonAtomicSync(
-          path(),
-          (cur) => {
-            if (Number(cur?.requestedAt) > requestedAt) return undefined;
-            written = true;
-            return {
-              ...extra,
-              ...(version != null ? { version } : {}),
-              fetchedAt: Date.now(),
-              requestedAt,
-              models,
-            };
-          },
-          { fsyncDir: true }
-        );
+        snapshot = JSON.parse(JSON.stringify({ extra, models }));
       } catch {
-        /* best-effort */
+        return Promise.resolve(false);
       }
-      if (written && onSave) onSave(models);
-      return written;
+      return enqueueWrite(p, async () => {
+        // A newer request may have started while this write waited its turn.
+        if (!stillWanted()) return false;
+        let written = false;
+        try {
+          await updateJsonAtomic(
+            p,
+            (cur) => {
+              if (Number(cur?.requestedAt) > requestedAt) return undefined;
+              // The lock wait is asynchronous: a newer request or a credential
+              // change may have landed since the checks above. Re-decide now,
+              // right before accepting the data.
+              if (!stillWanted()) return undefined;
+              written = true;
+              return {
+                ...snapshot.extra,
+                ...(version != null ? { version } : {}),
+                fetchedAt: Date.now(),
+                requestedAt,
+                models: snapshot.models,
+              };
+            },
+            { fsyncDir: true }
+          );
+        } catch {
+          /* best-effort */
+        }
+        try {
+          if (written && onSave) onSave(models);
+        } catch {
+          /* best-effort */
+        }
+        return written;
+      });
     };
   }
 

package/src/runtime/agent/orchestrator/providers/muse-code-oauth.mjs

@@ -435,7 +435,7 @@ export class MuseCodeOAuthProvider {
       });
     const rows = discovered.map((row) => museRow(row.id, row));
     // A superseded or other-credential response never replaces the live roster either.
-    if (publish(discovered)) inner._enrichedModels = rows;
+    if (await publish(discovered)) inner._enrichedModels = rows;
     return rows;
   }
 

package/src/runtime/agent/orchestrator/providers/oauth-secret-fields.fixtures.mjs

@@ -0,0 +1,45 @@
+// The authorization code a test login sends: redacted by value, since `code`
+// alone is also an error/status field.
+export const AUTH_CODE = 'SECRET-CODE';
+
+// Every secret field a token endpoint might echo, each with a distinct value
+// that a test can search for in pages, errors and logs.
+export const SECRET_FIELDS = {
+  access_token: 'SECRET-ACCESS',
+  refresh_token: 'SECRET-REFRESH',
+  id_token: 'SECRET-ID',
+  client_secret: 'SECRET-CLIENT',
+  code: AUTH_CODE,
+  code_verifier: 'SECRET-VERIFIER',
+  code_challenge: 'SECRET-CHALLENGE',
+  device_code: 'SECRET-DEVICE',
+  accessToken: 'SECRET-CAMEL-ACCESS',
+  refreshToken: 'SECRET-CAMEL-REFRESH',
+  idToken: 'SECRET-CAMEL-ID',
+  clientSecret: 'SECRET-CAMEL-CLIENT',
+  token: 'SECRET-TOKEN',
+};
+
+export const SECRET_RE = /SECRET-/;
+
+/** Valid JSON whose secret keys are written with escapes, nested in objects and arrays. */
+export const ESCAPED_SECRET_BODY = String.raw`{"error":{"code":400,"details":[{"client\u005fsecret":"SECRET-ESC-CLIENT","Refresh\u005FToken":"SECRET-ESC-REFRESH"},["x",{"\u0069d_token":"SECRET-ESC-ID"}]],"meta":{"api\u005Fkey":"SECRET-ESC-KEY","acc\u0065ss\u005ftoken":"SECRET-ESC-ACCESS"}}}`;
+
+/** Valid JSON whose secrets sit inside string values: form-encoded, a query string, nested JSON. */
+export const STRING_SECRET_BODY = JSON.stringify({
+  error: 'invalid_request',
+  detail: 'access_token=SECRET-FORM-ACCESS&client_secret=SECRET-FORM-CLIENT',
+  error_uri: 'https://auth.example/cb?client_secret=SECRET-QUERY-CLIENT&state=ok',
+  nested: JSON.stringify({ refresh_token: 'SECRET-NESTED', inner: String.raw`{"client\u005fsecret":"SECRET-DEEP"}` }),
+});
+
+/** Token-endpoint failure bodies carrying every secret field (JSON, escaped JSON, strings, form-encoded). */
+export const SECRET_BODIES = [
+  JSON.stringify({ error: 'invalid_grant', ...SECRET_FIELDS }),
+  ESCAPED_SECRET_BODY,
+  STRING_SECRET_BODY,
+  new URLSearchParams({ error: 'invalid_grant', ...SECRET_FIELDS }).toString(),
+];
+
+/** A 200 body that is not JSON but carries tokens (the parser would quote it). */
+export const MALFORMED_TOKEN_BODY = '{"access_token":"SECRET-ACCESS","refresh_token":"SECRET-REFRESH" oops';

package/src/runtime/agent/orchestrator/providers/openai-oauth-login.mjs

@@ -12,6 +12,7 @@ import { createServer } from 'node:http';
 import { randomBytes } from 'node:crypto';
 import { createOAuthPkce, parseOAuthCodeInput } from './lib/oauth-pkce.mjs';
 import { OAUTH_PAGE_CONTENT_TYPE, oauthErrorHtml, oauthSuccessHtml } from './lib/oauth-page.mjs';
+import { oauthFailureReason, oauthHttpError, oauthRequestError, oauthResponseJson } from './lib/oauth-token-utils.mjs';
 import { openLoginUrl } from './lib/open-login-url.mjs';
 import { sleepWithAbort } from './retry-backoff.mjs';
 
@@ -28,14 +29,6 @@ const TOKEN_TIMEOUT_MS = 30_000;
 const DEVICE_LOGIN_TIMEOUT_MS = 15 * 60_000;
 const DEFAULT_DEVICE_INTERVAL_S = 5;
 
-function _scrubOAuthLoginBody(text) {
-  return String(text || '')
-    .replace(/"access_token"\s*:\s*"[^"]+"/g, '"access_token":"[REDACTED]"')
-    .replace(/"refresh_token"\s*:\s*"[^"]+"/g, '"refresh_token":"[REDACTED]"')
-    .replace(/"id_token"\s*:\s*"[^"]+"/g, '"id_token":"[REDACTED]"')
-    .replace(/[A-Za-z0-9_-]{32,}\.[A-Za-z0-9._-]+/g, '[REDACTED]');
-}
-
 /**
  * @param {object} deps
  * @param {string} deps.clientId
@@ -72,26 +65,31 @@ export function createOpenAIOAuthLogin(deps) {
     signal?.throwIfAborted();
     const cleanCode = String(code || '').trim();
     if (!cleanCode) throw new Error('[openai-oauth] authorization code is required');
-    const tokenRes = await fetch(tokenUrl, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
-      body: new URLSearchParams({
-        grant_type: 'authorization_code',
-        code: cleanCode,
-        redirect_uri: redirectUri,
-        client_id: clientId,
-        code_verifier: pkce.verifier,
-      }),
-      redirect: 'error',
-      signal: signal
-        ? AbortSignal.any([signal, AbortSignal.timeout(TOKEN_TIMEOUT_MS)])
-        : AbortSignal.timeout(TOKEN_TIMEOUT_MS),
-    });
+    let tokenRes;
+    try {
+      tokenRes = await fetch(tokenUrl, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+        body: new URLSearchParams({
+          grant_type: 'authorization_code',
+          code: cleanCode,
+          redirect_uri: redirectUri,
+          client_id: clientId,
+          code_verifier: pkce.verifier,
+        }),
+        redirect: 'error',
+        signal: signal
+          ? AbortSignal.any([signal, AbortSignal.timeout(TOKEN_TIMEOUT_MS)])
+          : AbortSignal.timeout(TOKEN_TIMEOUT_MS),
+      });
+    } catch (err) {
+      throw oauthRequestError('[openai-oauth] token exchange', err);
+    }
     if (!tokenRes.ok) {
       const text = await tokenRes.text().catch(() => '');
-      throw new Error(`[openai-oauth] token exchange ${tokenRes.status}: ${_scrubOAuthLoginBody(text).slice(0, 500)}`);
+      throw oauthHttpError('[openai-oauth] token exchange', tokenRes.status, text, [cleanCode, pkce.verifier]);
     }
-    const json = await tokenRes.json();
+    const json = await oauthResponseJson(tokenRes, '[openai-oauth] token exchange');
     if (!json.access_token || !json.refresh_token) {
       throw new Error('[openai-oauth] token exchange response missing access_token or refresh_token');
     }
@@ -292,10 +290,10 @@ export function createOpenAIOAuthLogin(deps) {
           // written; do not leave a keep-alive connection bound to it.
           res.writeHead(200, { 'Content-Type': OAUTH_PAGE_CONTENT_TYPE, Connection: 'close' });
           res.end(oauthSuccessHtml('OpenAI'));
-        } catch {
-          // The shared exchange reports its error through waitForCallback.
+        } catch (err) {
+          // The shared exchange also reports its error through waitForCallback.
           res.writeHead(500, { 'Content-Type': OAUTH_PAGE_CONTENT_TYPE, Connection: 'close' });
-          res.end(oauthErrorHtml('OpenAI could not complete the token exchange.'));
+          res.end(oauthErrorHtml(oauthFailureReason('OpenAI could not complete the token exchange.', err)));
         }
       });
       timeout = setTimeout(() => finish(null), LOGIN_TIMEOUT_MS);

package/src/runtime/agent/orchestrator/providers/openai-ws-stream/event-switch.mjs

@@ -40,7 +40,9 @@ export function createWsEventSwitch({
   // progress only once the model has produced output; before that the silent
   // thinking follows it.
   function onItemAdded() {
-    const kind = response.toolInFlight ? 'tool' : watchdogs.semanticSeen() ? 'semantic' : null;
+    let kind = null;
+    if (response.toolInFlight) kind = 'tool';
+    else if (watchdogs.semanticSeen()) kind = 'semantic';
     if (kind) watchdogs.bumpSemantic();
     progress(kind ?? 'transport');
   }

package/src/runtime/agent/orchestrator/providers/openai-ws-watchdogs.mjs

@@ -93,6 +93,7 @@ export function createWsStreamWatchdogs({
       // synthesized completion. Attach the streamed partial state so
       // the loop's partial/tool recovery can finalize it explicitly.
       err.streamStalled = true;
+      err.code = 'ESTREAMSTALL';
       attachPartialState(err);
       fail(err, 'inter_chunk_timeout');
     }, interChunkMs);

package/src/runtime/agent/orchestrator/providers/opencode-go-oauth.mjs

@@ -228,7 +228,7 @@ export class OpenCodeGoOAuthProvider {
     });
     const response = await consoleGetJson(CONFIG_URL, tokens, { fetchFn: this.config.fetchImpl, refresh: true });
     const remote = goConfigFromResponse(response, listsGoModels(response) ? null : await loadModelsDevGoModels());
-    publish(remote.models, {
+    await publish(remote.models, {
       baseURL: remote.baseURL,
       headers: remote.headers,
       accountId: tokens.account_id,

package/src/runtime/agent/orchestrator/providers/opencode-go.mjs

@@ -170,7 +170,9 @@ export class OpenCodeGoProvider {
       const { OpenCodeGoGeminiEngine } = await import('./opencode-go-gemini.mjs');
       this.gemini = new OpenCodeGoGeminiEngine({
         ...this.config,
-        baseURL: this.openai.baseURL,
+        // The OAuth org config names the chat base `.../go/openai/v1`; the
+        // generateContent routes live under `.../go/v1` and 404 on it.
+        baseURL: String(this.openai.baseURL).replace(/\/openai\/v1$/i, '/v1'),
         extraHeaders: {
           ...GO_GATEWAY_PRESET.extraHeaders,
           ...(this.config.extraHeaders || {}),

package/src/runtime/agent/orchestrator/providers/registry.mjs

@@ -3,7 +3,9 @@ import { oauthCredentialProbeState } from './oauth-credential-probes.mjs';
 import { providerAvailable } from './provider-availability.mjs';
 import { refreshCatalog as refreshMetadataCatalog } from './model-catalog.mjs';
 import { wrapProviderAdmission } from './admission-scheduler.mjs';
-import { createAccountPoolProvider } from './account-pool.mjs';
+import { createAccountPoolProvider, createCustomKeyPoolProvider } from './account-pool.mjs';
+import { customKeyEngine } from './custom-provider-keys.mjs';
+import { customProviderKeyIds } from '../../../shared/provider-accounts.mjs';
 import { isInclusiveProvider } from '../../../shared/llm/cost.mjs';
 import { noEnabledProvidersError } from './registry-errors.mjs';
 import { providerTakesNativePdf } from '../../../attachments/store.mjs';
@@ -182,6 +184,7 @@ async function loadProviderExport(cacheKey, spec, exportName, signal = null) {
 const PROVIDER_MODULES = {
   anthropic: ['./anthropic.mjs', 'AnthropicProvider'],
   gemini: ['./gemini.mjs', 'GeminiProvider'],
+  'vertex-ai': ['./vertex-ai.mjs', 'VertexAIProvider'],
   'openai-oauth': ['./openai-oauth.mjs', 'OpenAIOAuthProvider'],
   'anthropic-oauth': ['./anthropic-oauth.mjs', 'AnthropicOAuthProvider'],
   'grok-oauth': ['./grok-oauth.mjs', 'GrokOAuthProvider'],
@@ -227,13 +230,39 @@ async function instantiateProvider(name, Ctor, cfg, proxyChannel = null) {
     const { createModelProxyProvider } = await import('./model-proxy-provider.mjs');
     return wrapProviderAdmission(createModelProxyProvider(name, cfg, proxyChannel ?? getModelProxyChannel()), name);
   }
-  if (isCustomProviderConfig(name, cfg)) return wrapProviderAdmission(await Ctor(name, cfg), name);
+  if (isCustomProviderConfig(name, cfg)) {
+    const { [CUSTOM_KEY_IDS]: keyIds, ...settings } = cfg;
+    const primary = wrapProviderAdmission(await Ctor(name, settings), name);
+    // Only a provider with additional keys sends through the key roster.
+    if (!keyIds?.length) return primary;
+    return createCustomKeyPoolProvider(name, primary, (id) => customKeyEngine(primary, name, id));
+  }
   if (Object.hasOwn(OPENAI_COMPAT_PRESETS, name) && !Object.hasOwn(PROVIDER_MODULES, name)) {
     return wrapProviderAdmission(new Ctor(name, cfg), name);
   }
   return createAccountPoolProvider(name, () => wrapProviderAdmission(new Ctor(cfg), name));
 }
 
+// A custom provider's additional keys live in its key roster, not the config.
+// Their ids are read once per init and join its entry (and so its reuse
+// signature): a save that changes them rebuilds the provider, which is wrapped
+// in the key pool only while it has some. A single-key provider's entry and
+// sends stay as they were, with no roster read per send.
+const CUSTOM_KEY_IDS = 'customKeyIds';
+
+function withCustomKeyIds(config) {
+  if (!config || typeof config !== 'object') return config;
+  let out = config;
+  for (const [name, cfg] of Object.entries(config)) {
+    if (!isCustomProviderConfig(name, cfg) || cfg.auth === 'none') continue;
+    const ids = customProviderKeyIds(name);
+    if (!ids.length) continue;
+    if (out === config) out = { ...config };
+    out[name] = { ...cfg, [CUSTOM_KEY_IDS]: ids };
+  }
+  return out;
+}
+
 let _initProvidersCalls = 0;
 /** Test seam: how many times initProviders was called in this process. */
 export function _initProvidersCallCountForTest() {
@@ -249,6 +278,8 @@ export async function initProviders(config, { signal = null } = {}) {
   if (proxyChannel) {
     const { loadModelProxyProviderConfig } = await import('./model-proxy-provider.mjs');
     config = await awaitWithAbort(loadModelProxyProviderConfig({ channel: proxyChannel }), signal);
+  } else {
+    config = withCustomKeyIds(config);
   }
   const sig = reuseSignature(config, proxyChannel);
   // Coalesce: an identical config is already mid-init — attach to it.