mixdog 0.9.172 → 0.9.173

package/src/runtime/media/auth.mjs

@@ -11,6 +11,15 @@ export { hasGrokOAuthCredentials } from '../agent/orchestrator/providers/oauth-c
 
 const XAI_BASE_URL = 'https://api.x.ai/v1';
 
+// Built here rather than with lanes.mjs's mediaError: importing it would close
+// an auth -> lanes -> catalog -> auth module cycle.
+function unauthenticated(message) {
+  const err = new Error(message);
+  err.code = 'MEDIA_LANE_UNAUTHENTICATED';
+  err.status = 401;
+  return err;
+}
+
 /**
  * Bearer for the xAI media endpoints. `grok-oauth` refreshes through the chat
  * provider; `xai` uses the stored API key. Both hit the same api.x.ai routes.
@@ -18,18 +27,17 @@ const XAI_BASE_URL = 'https://api.x.ai/v1';
 export async function resolveXaiAuth(laneId) {
   if (laneId === 'xai') {
     const key = getAgentApiKey('xai');
-    if (!key) {
-      const err = new Error('xAI API key is not configured');
-      err.code = 'MEDIA_LANE_UNAUTHENTICATED';
-      err.status = 401;
-      throw err;
-    }
+    if (!key) throw unauthenticated('xAI API key is not configured');
     return { baseURL: XAI_BASE_URL, token: key };
   }
   const { GrokOAuthProvider } = await import('../agent/orchestrator/providers/grok-oauth.mjs');
   const provider = new GrokOAuthProvider({});
   const tokens = await provider.ensureAuth();
-  return { baseURL: XAI_BASE_URL, token: tokens.access_token };
+  // The bearer rotates on every refresh, so it cannot identify this credential
+  // across sessions. The account id can, exactly like Antigravity's Cloud
+  // project: without it the catalog cache re-keyed on each refresh, re-fetched
+  // on every Studio entry, and lost its own offline fallback.
+  return { baseURL: XAI_BASE_URL, token: tokens.access_token, accountId: tokens.user_id || '' };
 }
 
 /** Codex (ChatGPT OAuth) auth record: access token + account id for headers. */
@@ -53,11 +61,6 @@ export async function resolveAntigravityAuth() {
 
 export function resolveGeminiKey() {
   const key = getAgentApiKey('gemini');
-  if (!key) {
-    const err = new Error('Gemini API key is not configured');
-    err.code = 'MEDIA_LANE_UNAUTHENTICATED';
-    err.status = 401;
-    throw err;
-  }
+  if (!key) throw unauthenticated('Gemini API key is not configured');
   return key;
 }

package/src/runtime/media/catalog.mjs

@@ -34,10 +34,15 @@ function googleLane(lane) {
   return lane === 'gemini' || lane === 'antigravity-oauth';
 }
 
+// Both xAI lanes (stored API key and Grok OAuth) speak the same Imagine catalog.
+function grokLane(lane) {
+  return lane === 'xai' || lane === 'grok-oauth';
+}
+
 function kindFor(lane, row, id) {
   if (row.deprecated === true || row.disabled === true) return null;
   if (lane === 'openai-oauth') return imageToolSupported(row, id) ? 'image' : null;
-  if (lane === 'xai' || lane === 'grok-oauth') {
+  if (grokLane(lane)) {
     if (/^grok-imagine-image(?:-|$)/.test(id)) return 'image';
     // This version requires a start image. Studio also permits prompt-only
     // generation, so keep it out until the required-input contract is exposed.
@@ -79,7 +84,7 @@ function compareModels(a, b) {
 /** Keep official tiers and generations intact; never infer a quality ranking. */
 function mediaModelLabel(lane, row, id) {
   let label = String(row.displayName || row.display || row.label || row.name || id).replace(/^models\//, '');
-  if (lane === 'xai' || lane === 'grok-oauth') {
+  if (grokLane(lane)) {
     label = id
       .replace(/^grok-imagine-image/, 'Grok Imagine Image')
       .replace(/^grok-imagine-video/, 'Grok Imagine Video')
@@ -112,7 +117,7 @@ export function projectMediaModels(lane, rows) {
     const kind = kindFor(lane, row, id);
     if (!kind) continue;
     let controls = {};
-    if ((lane === 'xai' || lane === 'grok-oauth') && kind === 'video') {
+    if (grokLane(lane) && kind === 'video') {
       controls = { resolution: ['480p', '720p'] };
     } else if (lane === 'gemini' && /^gemini-omni-/.test(id)) {
       controls = { resolution: [], durations: [], maxReferences: 3 };
@@ -177,10 +182,13 @@ async function providerSource(lane) {
   else if (lane === 'antigravity-oauth') auth = await resolveAntigravityAuth();
   else auth = await resolveXaiAuth(lane);
   // API-key and OAuth catalogs must not leak availability across credentials.
-  // Persist only a one-way scope hash, never a key or a bearer. Antigravity
-  // bearers rotate hourly; the Cloud project identifies that account instead.
+  // Persist only a one-way scope hash, never a key or a bearer. OAuth bearers
+  // rotate (Antigravity hourly, Grok on every refresh), so they must never be
+  // the scope: a rotating key means a permanent cache miss AND a stale-cache
+  // fallback that can no longer find its own previous catalog. The stable
+  // account identity — Cloud project or account id — plays that role instead.
   const scope = createHash('sha256')
-    .update(auth.projectId || auth.token)
+    .update(auth.projectId || auth.accountId || auth.token)
     .digest('hex')
     .slice(0, 24);
   return {

package/src/runtime/media/lanes.mjs

@@ -17,8 +17,9 @@ import {
 } from '../agent/orchestrator/providers/oauth-credential-probes.mjs';
 import { loadMediaModels, projectMediaModels } from './catalog.mjs';
 import { catalogDiagnostic } from './catalog-errors.mjs';
+import { MEDIA_KINDS } from './tool-defs.mjs';
 
-export const MEDIA_KINDS = Object.freeze(['image', 'video']);
+export { MEDIA_KINDS };
 
 // Aspect/resolution vocabularies are lane-native: xAI takes aspect_ratio +
 // resolution, Gemini/Codex take pixel sizes. The UI renders whatever the lane

package/src/runtime/memory/index.mjs

@@ -31,7 +31,7 @@ import { TOOL_DEFS } from './tool-defs.mjs';
 
 // Static import (not the dynamic one in stop()) so the sync stop is available
 // inside a process 'exit' hook, where dynamic import() cannot run.
-import { stopPgForShutdownSync } from './lib/pg/supervisor.mjs';
+import { stopPgForShutdown, stopPgForShutdownSync } from './lib/pg/supervisor.mjs';
 
 import {
   openDatabase,
@@ -51,7 +51,6 @@ import {
   shutdownEmbeddingProvider,
   warmupEmbeddingProvider,
 } from './lib/embedding-provider.mjs';
-import { startLlmWorker, stopLlmWorker } from './lib/llm-worker-host.mjs';
 import { runCycle1, runCycle2, parseInterval, flushEmbeddingDirty, flushRawEmbeddings } from './lib/memory-cycle.mjs';
 import { callAgentDispatch } from './lib/agent-ipc.mjs';
 import {
@@ -139,12 +138,11 @@ import {
 } from '../shared/service-discovery.mjs';
 
 const MEMORY_SERVER_PID = parsePositivePid(process.env.MIXDOG_SERVER_PID) ?? process.pid;
-const _isPidAliveLocal = isPidAliveLocal;
 const _memoryPortAdvertiser = createMemoryPortAdvertiser({
   readServiceAdvert: _readServiceAdvert,
   writeServiceAdvert: _writeServiceAdvert,
   parsePositivePid,
-  isPidAliveLocal: _isPidAliveLocal,
+  isPidAliveLocal,
   memoryServerPid: MEMORY_SERVER_PID,
   log: __mixdogMemoryLog,
 });
@@ -157,7 +155,7 @@ const _daemonLifecycle = createMemoryDaemonLifecycle({
   idleTtlMs: MEMORY_IDLE_TTL_MS,
   clientGraceMs: MEMORY_CLIENT_GRACE_MS,
   parsePositivePid,
-  isPidAlive: _isPidAliveLocal,
+  isPidAlive: isPidAliveLocal,
   isStopping: () => _serviceLifecycle.isStopping(),
   stop,
   log: __mixdogMemoryLog,
@@ -318,9 +316,7 @@ async function _initStore() {
   if (!(await isBootstrapComplete(db))) {
     throw new Error('memory-service: bootstrap not complete after openDatabase');
   }
-  if (memoryLlmWorkerEnabled()) {
-    startLlmWorker();
-  } else {
+  if (!memoryLlmWorkerEnabled()) {
     __mixdogMemoryLog('[memory-service] secondary mode; skipping llm worker\n');
   }
   // Provider/session/agent modules live once in the daemon. Memory
@@ -673,7 +669,7 @@ async function _stopBackgroundWork() {
   _stopCycles();
   _embeddingReindexController?.abort(new Error('memory service stopping'));
   const reindexPromise = _embeddingReindexPromise;
-  await Promise.allSettled([stopLlmWorker(), reindexPromise]);
+  await Promise.allSettled([reindexPromise]);
   await shutdownEmbeddingProvider();
 }
 
@@ -689,7 +685,6 @@ async function _stopRuntime() {
   // closeDatabase() only ends the client pool; without this the child
   // postmaster keeps running after the unified daemon exits.
   if (!memorySecondaryMode()) {
-    const { stopPgForShutdown } = await import('./lib/pg/supervisor.mjs');
     await stopPgForShutdown();
   } else {
     __mixdogMemoryLog('[memory-service] secondary mode; leaving shared PG running\n');

package/src/runtime/memory/lib/core-memory-store.mjs

@@ -1,5 +1,4 @@
 import { __mixdogMemoryLog } from './memory-log.mjs';
-export { __mixdogMemoryLog };
 
 // User-curated core memory store — native PG-backed via core_entries table.
 // Per-project entries distinguished by project_id column (NULL = COMMON).
@@ -7,13 +6,12 @@ export { __mixdogMemoryLog };
 // Generated conversation history never enters this store automatically.
 
 import { getDatabase, embeddingToSql } from './memory.mjs';
+import { VALID_CATEGORY } from './memory-categories.mjs';
 import { cachedEmbedTextBatch } from './memory-embed.mjs';
 import { checkedConnect } from './pg/adapter.mjs';
 import { throwIfAborted } from './memory-cycle2-shared.mjs';
 import { findCoreKeyRows } from './core-memory-uniqueness.mjs';
 
-const VALID_CAT = new Set(['rule', 'constraint', 'decision', 'fact', 'goal', 'preference', 'task', 'issue']);
-
 const CORE_ELEMENT_DERIVE_LENGTH = 40;
 
 // Op aliases: the surface verbs are add/edit/delete/list/…, and a caller
@@ -63,8 +61,8 @@ export function normalizeCoreInput(input = {}, options = {}) {
   if (options.requireElement && !element) errors.push('element required');
   if (options.requireSummary && !summary) errors.push('summary required');
   if (options.requireCategory && !suppliedCategory) errors.push('category required');
-  if (suppliedCategory && !VALID_CAT.has(category)) {
-    errors.push(`invalid category "${category}". Valid: ${[...VALID_CAT].join(', ')}`);
+  if (suppliedCategory && !VALID_CATEGORY.has(category)) {
+    errors.push(`invalid category "${category}". Valid: ${[...VALID_CATEGORY].join(', ')}`);
   }
 
   return { element, summary, category, suppliedCategory, errors };
@@ -87,8 +85,6 @@ async function _embedFor(db, element, summary) {
 // Lazy repair of NULL embeddings on existing rows. Runs once per boot or
 // whenever a NULL slips back in via direct SQL. SELECT WHERE embedding IS NULL
 // returns 0 rows on a fully-populated table, so this is a fast no-op.
-export { throwIfAborted };
-
 async function _backfillNullEmbeddings(db, options = {}) {
   const signal = options?.signal;
   throwIfAborted(signal);

package/src/runtime/memory/lib/embedding-provider.mjs

@@ -53,14 +53,6 @@ export function embeddingWorkerExecArgv(execArgv = process.execArgv) {
   });
 }
 
-function cacheEmbedding(key, vector) {
-  queryEmbeddingCache.set(key, vector);
-}
-
-function getCachedEmbedding(key) {
-  return queryEmbeddingCache.get(key);
-}
-
 function embeddingCacheKey(text, inputType, dtype = _configuredDtype) {
   return `${MODEL_ID}\n${dtype}\n${normalizeEmbeddingInputType(inputType)}\n${text}`;
 }
@@ -307,7 +299,7 @@ export async function embedText(text, options = {}) {
   const inputType = normalizeEmbeddingInputType(options?.inputType);
   const dtype = _configuredDtype;
   const cacheKey = embeddingCacheKey(clean, inputType, dtype);
-  const cached = getCachedEmbedding(cacheKey);
+  const cached = queryEmbeddingCache.get(cacheKey);
   if (cached) return [...cached];
 
   // Interactive query embeds pass { priority: true } so the worker can
@@ -331,7 +323,7 @@ export async function embedText(text, options = {}) {
   if (!Array.isArray(vector) || vector.length !== cachedDims) {
     throw new Error(`embed vector length mismatch: expected ${cachedDims}, got ${vector?.length}`);
   }
-  cacheEmbedding(cacheKey, vector);
+  queryEmbeddingCache.set(cacheKey, vector);
   _embedCallCount++;
   if (_embedCallCount % EMBED_STEADY_SAMPLE_EVERY === 0) {
     writeProfilePoint({
@@ -365,11 +357,13 @@ export async function embedTexts(texts, options = {}) {
     const key = embeddingCacheKey(t, inputType, dtype);
     if (!queryEmbeddingCache.has(key)) missing.push(t);
   }
-  if (missing.length === 0)
-    return cleaned.map((t) => {
+  const cachedResults = () =>
+    cleaned.map((t) => {
       if (!t) return [];
-      return [...queryEmbeddingCache.get(embeddingCacheKey(t, inputType, dtype))];
+      const cached = queryEmbeddingCache.get(embeddingCacheKey(t, inputType, dtype));
+      return cached ? [...cached] : [];
     });
+  if (missing.length === 0) return cachedResults();
   const result = await sendToWorker('embed-batch', { texts: missing, inputType });
   if (!result.dims) throw new Error(`embed-batch result missing dims (model=${MODEL_ID})`);
   const resultDims = result.dims;
@@ -389,14 +383,10 @@ export async function embedTexts(texts, options = {}) {
     if (!Array.isArray(vec) || vec.length !== cachedDims) {
       throw new Error(`embed-batch vector length mismatch at idx ${i}: expected ${cachedDims}, got ${vec?.length}`);
     }
-    cacheEmbedding(embeddingCacheKey(missing[i], inputType, dtype), vec);
+    queryEmbeddingCache.set(embeddingCacheKey(missing[i], inputType, dtype), vec);
   }
   _embedCallCount += missing.length;
-  return cleaned.map((t) => {
-    if (!t) return [];
-    const cached = queryEmbeddingCache.get(embeddingCacheKey(t, inputType, dtype));
-    return cached ? [...cached] : [];
-  });
+  return cachedResults();
 }
 
 export async function shutdownEmbeddingProvider() {

package/src/runtime/memory/lib/embedding-worker.mjs

@@ -206,6 +206,26 @@ async function disposeLoadedExtractor(reason) {
   }
 }
 
+function cancelIdleTimer() {
+  if (!_idleTimer) return;
+  clearTimeout(_idleTimer);
+  _idleTimer = null;
+}
+
+// Tear down the loaded extractor for a control op (configure/dispose). Both
+// hold the in-flight guard, so no inference can observe the reset mid-flight.
+async function disposeExtractorForControlOp() {
+  if (!extractorPromise) return;
+  try {
+    const ext = await extractorPromise;
+    try {
+      ext.dispose();
+    } catch {}
+  } catch {}
+  extractorPromise = null;
+  _device = 'cpu';
+}
+
 function resetIdleTimer() {
   if (_idleTimer) clearTimeout(_idleTimer);
   if (IDLE_TIMEOUT_MS <= 0) return;
@@ -498,23 +518,11 @@ async function processMessage(msg) {
           return;
         }
         _embedInFlight = true;
-        if (_idleTimer) {
-          clearTimeout(_idleTimer);
-          _idleTimer = null;
-        }
+        cancelIdleTimer();
         if (msg.dtype != null) {
           configuredDtype = normalizeEmbeddingDtype(MODEL_ID, msg.dtype);
         }
-        if (extractorPromise) {
-          try {
-            const ext = await extractorPromise;
-            try {
-              ext.dispose();
-            } catch {}
-          } catch {}
-          extractorPromise = null;
-          _device = 'cpu';
-        }
+        await disposeExtractorForControlOp();
         parentPort.postMessage({ id, type: 'result' });
         break;
       }
@@ -524,21 +532,9 @@ async function processMessage(msg) {
           return;
         }
         _embedInFlight = true;
-        if (_idleTimer) {
-          clearTimeout(_idleTimer);
-          _idleTimer = null;
-        }
+        cancelIdleTimer();
         const prevDevice = _device;
-        if (extractorPromise) {
-          try {
-            const ext = await extractorPromise;
-            try {
-              ext.dispose();
-            } catch {}
-          } catch {}
-          extractorPromise = null;
-          _device = 'cpu';
-        }
+        await disposeExtractorForControlOp();
         parentPort.postMessage({ id, type: 'result', prevDevice, dtype: configuredDtype });
         break;
       }

package/src/runtime/memory/lib/http-router/admin-routes.mjs

@@ -5,7 +5,7 @@
  */
 import { readBody, sendJson, normalizeCoreProjectId } from '../http-wire.mjs';
 import { listCore, addCore, deleteCore } from '../core-memory-store.mjs';
-import { openTraceDatabase, insertAgentCalls, enqueueTraceEvents, registerTraceExitDrain } from '../trace-store.mjs';
+import { createAdminTraceRoutes } from './admin-trace-routes.mjs';
 
 // One admin route: any thrown error becomes `{ ok:false, error }` 500.
 const guarded = (handle) => async (req, res) => {
@@ -128,55 +128,9 @@ export function createAdminRoutes({
     sendJson(res, { ok: true, deleted: preCount, core_preserved: coreCount });
   });
 
-  // The trace DB opens lazily on the first record; null means tracing is
-  // disabled for this data dir.
-  const openTraceDb = async () => {
-    const existing = getTraceDb();
-    if (existing) return existing;
-    const traceDb = await openTraceDatabase(dataDir);
-    if (!traceDb) return null;
-    setTraceDb(traceDb);
-    registerTraceExitDrain(traceDb);
-    return traceDb;
-  };
-  const traceRecord = async (req, res) => {
-    let body;
-    try {
-      body = await readBody(req);
-    } catch (e) {
-      sendJson(res, { ok: false, error: e.message }, 400);
-      return;
-    }
-    if (!Array.isArray(body?.events)) {
-      sendJson(res, { ok: false, error: 'body.events must be an array' }, 400);
-      return;
-    }
-    if (body.events.length > 500) {
-      sendJson(res, { ok: false, error: 'too many events (max 500)' }, 413);
-      return;
-    }
-    let traceDb;
-    try {
-      traceDb = await openTraceDb();
-    } catch (e) {
-      sendJson(res, { ok: false, error: `trace DB unavailable: ${e.message}` }, 503);
-      return;
-    }
-    if (!traceDb) {
-      sendJson(res, { ok: true, queued: 0, disabled: true });
-      return;
-    }
-    try {
-      // Enqueue for async batched flush (100ms / 500-row window).
-      enqueueTraceEvents(traceDb, body.events);
-      // Use `queued` — events are async; `inserted` would imply durability.
-      sendJson(res, { ok: true, queued: body.events.length });
-      // Fire-and-forget into focused agent analytic tables.
-      insertAgentCalls(traceDb, body.events).catch((e) => log(`[trace] insertAgentCalls error: ${e?.message}\n`));
-    } catch (e) {
-      sendJson(res, { ok: false, error: e.message }, 500);
-    }
-  };
+  // /admin/trace-record and its lazily opened database
+  // (admin-trace-routes.mjs).
+  const traceRoutes = createAdminTraceRoutes({ dataDir, log, getTraceDb, setTraceDb });
 
   const shutdown = (_req, res) => {
     sendJson(res, { shutting_down: true }, 202);
@@ -207,7 +161,7 @@ export function createAdminRoutes({
     'POST /admin/entries/add': addEntry,
     'POST /admin/backfill': backfill,
     'POST /admin/purge': purge,
-    'POST /admin/trace-record': traceRecord,
+    ...traceRoutes,
     'POST /admin/shutdown': shutdown,
   };
 }

package/src/runtime/memory/lib/http-router/admin-trace-routes.mjs

@@ -0,0 +1,65 @@
+/**
+ * http-router/admin-trace-routes.mjs — the /admin/trace-record surface: the
+ * trace database opened lazily on the first record (and left null when tracing
+ * is disabled for this data dir), the request validation and batch size cap,
+ * the async batched enqueue, and the fire-and-forget agent-analytics insert.
+ *
+ * The database handle itself belongs to the server, which passes the
+ * getTraceDb/setTraceDb accessors, so the exit drain is registered once.
+ */
+import { readBody, sendJson } from '../http-wire.mjs';
+import { openTraceDatabase, insertAgentCalls, enqueueTraceEvents, registerTraceExitDrain } from '../trace-store.mjs';
+
+export function createAdminTraceRoutes({ dataDir, log, getTraceDb, setTraceDb }) {
+  // The trace DB opens lazily on the first record; null means tracing is
+  // disabled for this data dir.
+  const openTraceDb = async () => {
+    const existing = getTraceDb();
+    if (existing) return existing;
+    const traceDb = await openTraceDatabase(dataDir);
+    if (!traceDb) return null;
+    setTraceDb(traceDb);
+    registerTraceExitDrain(traceDb);
+    return traceDb;
+  };
+  const traceRecord = async (req, res) => {
+    let body;
+    try {
+      body = await readBody(req);
+    } catch (e) {
+      sendJson(res, { ok: false, error: e.message }, 400);
+      return;
+    }
+    if (!Array.isArray(body?.events)) {
+      sendJson(res, { ok: false, error: 'body.events must be an array' }, 400);
+      return;
+    }
+    if (body.events.length > 500) {
+      sendJson(res, { ok: false, error: 'too many events (max 500)' }, 413);
+      return;
+    }
+    let traceDb;
+    try {
+      traceDb = await openTraceDb();
+    } catch (e) {
+      sendJson(res, { ok: false, error: `trace DB unavailable: ${e.message}` }, 503);
+      return;
+    }
+    if (!traceDb) {
+      sendJson(res, { ok: true, queued: 0, disabled: true });
+      return;
+    }
+    try {
+      // Enqueue for async batched flush (100ms / 500-row window).
+      enqueueTraceEvents(traceDb, body.events);
+      // Use `queued` — events are async; `inserted` would imply durability.
+      sendJson(res, { ok: true, queued: body.events.length });
+      // Fire-and-forget into focused agent analytic tables.
+      insertAgentCalls(traceDb, body.events).catch((e) => log(`[trace] insertAgentCalls error: ${e?.message}\n`));
+    } catch (e) {
+      sendJson(res, { ok: false, error: e.message }, 500);
+    }
+  };
+
+  return { 'POST /admin/trace-record': traceRecord };
+}