mixdog 0.9.151 → 0.9.153

package/src/runtime/computer-bridge/client.mjs

@@ -0,0 +1,231 @@
+/**
+ * Loopback client for the desktop app's computer-use bridge.
+ *
+ * Mirrors browser-bridge/client.mjs: the Mixdog desktop app serves computer
+ * commands on 127.0.0.1 and advertises { port, token } through a heartbeated
+ * discovery file. This runtime half is a sync availability probe that gates the
+ * `computer` tool surface plus the async executor behind tool calls. The bridge
+ * only exists while the desktop app runs with Computer Use enabled.
+ */
+import { readFileSync, statSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import {
+  toComputerHostCommand,
+  validateComputerToolArgs,
+} from './action-schema.mjs';
+
+const DISCOVERY_FILE = 'computer-bridge.json';
+const DISCOVERY_VERSION = 1;
+const DISCOVERY_MAX_AGE_MS = 5 * 60_000;
+// Desktop UI Automation queries and input dispatch can be slow; sit above the
+// bridge's own per-action timeouts so its specific error wins over a bare abort.
+const REQUEST_TIMEOUT_MS = 150_000;
+const SESSION_ABORT_TIMEOUT_MS = 8_000;
+const SESSION_RELEASE_TIMEOUT_MS = 60_000;
+const DEFERRED_SESSION_RELEASE_MS = 2 * 60_000;
+const READ_ONLY_ACTIONS = new Set([
+  'list_windows', 'list_apps', 'diagnose', 'capture', 'snapshot', 'find', 'clipboard_read', 'wait',
+  'window_bounds', 'screenshot', 'zoom',
+]);
+const activeComputerSessions = new Set();
+const deferredComputerSessionReleases = new Map();
+
+const BRIDGE_UNAVAILABLE_MESSAGE =
+  'computer use is unavailable; open the Mixdog desktop app and enable Computer Use in settings';
+
+function canonicalResultText(text, args) {
+  let value;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    return text;
+  }
+  if (!value || typeof value !== 'object' || Array.isArray(value)
+    || typeof value.action !== 'string') {
+    return text;
+  }
+  value.action = args.action;
+  if (args.action === 'list') value.kind = args.input?.kind;
+  if (args.action === 'capture') value.mode = args.input?.mode || 'state';
+  if (args.action === 'click') value.button = args.input?.button || 'left';
+  if (args.action === 'window') value.operation = args.input?.operation;
+  if (args.action === 'clipboard') value.operation = args.input?.operation;
+  return JSON.stringify(value);
+}
+
+function discoveryPath() {
+  const dataDir = process.env.MIXDOG_DATA_DIR
+    || join(process.env.MIXDOG_HOME || join(homedir(), '.mixdog'), 'data');
+  return join(dataDir, DISCOVERY_FILE);
+}
+
+/** Sync gate for the session tool surface (featureDisallowedTools). */
+export function computerBridgeAvailableSync() {
+  return readDiscovery() !== null;
+}
+
+function readDiscovery() {
+  let parsed;
+  try {
+    const path = discoveryPath();
+    if (Date.now() - statSync(path).mtimeMs >= DISCOVERY_MAX_AGE_MS) return null;
+    parsed = JSON.parse(readFileSync(path, 'utf8'));
+  } catch {
+    return null;
+  }
+  const version = Number(parsed?.version);
+  const port = Number(parsed?.port);
+  const token = String(parsed?.token || '');
+  if (version !== DISCOVERY_VERSION
+    || !Number.isInteger(port) || port <= 0 || port > 65_535 || !token) return null;
+  return { port, token };
+}
+
+/** Execute one `computer` tool call. Returns MCP-shaped content so the
+ *  internal-tools normalizer forwards text and screenshot images as-is. */
+export async function executeComputerTool(args, context = {}) {
+  const discovery = readDiscovery();
+  if (!discovery) {
+    return { content: [{ type: 'text', text: `Error: ${BRIDGE_UNAVAILABLE_MESSAGE}` }], isError: true };
+  }
+  const validationError = validateComputerToolArgs(args);
+  if (validationError) {
+    return { content: [{ type: 'text', text: `Error: ${validationError}` }], isError: true };
+  }
+  let response;
+  const command = toComputerHostCommand(args);
+  const sessionId = context?.sessionId ? String(context.sessionId) : '';
+  cancelDeferredComputerSessionRelease(sessionId);
+  const action = String(command?.action || '');
+  if (sessionId && !READ_ONLY_ACTIONS.has(action) && command?.read_only !== true) {
+    activeComputerSessions.add(sessionId);
+  }
+  try {
+    const timeoutSignal = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
+    const requestSignal = context?.signal
+      ? AbortSignal.any([timeoutSignal, context.signal])
+      : timeoutSignal;
+    response = await fetch(`http://127.0.0.1:${discovery.port}/command`, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        authorization: `Bearer ${discovery.token}`,
+      },
+      body: JSON.stringify({
+        ...command,
+        ...(context?.sessionId ? { session_id: String(context.sessionId) } : {}),
+      }),
+      signal: requestSignal,
+    });
+  } catch (error) {
+    const externallyAborted = context?.signal?.aborted === true;
+    const abortConfirmed = externallyAborted && sessionId
+      ? await abortComputerSession(sessionId)
+      : false;
+    const reason = error?.name === 'TimeoutError'
+      ? 'computer bridge timed out'
+      : externallyAborted
+        ? (abortConfirmed
+            ? 'computer command aborted; input state and session resources were released'
+            : 'computer command aborted; host cleanup could not be confirmed')
+        : BRIDGE_UNAVAILABLE_MESSAGE;
+    return { content: [{ type: 'text', text: `Error: ${reason}` }], isError: true };
+  }
+  let body;
+  try {
+    body = await response.json();
+  } catch {
+    return { content: [{ type: 'text', text: `Error: computer bridge returned an invalid response (HTTP ${response.status})` }], isError: true };
+  }
+  if (!body?.ok) {
+    const message = String(body?.error || `computer bridge request failed (HTTP ${response.status})`);
+    return { content: [{ type: 'text', text: message.startsWith('Error:') ? message : `Error: ${message}` }], isError: true };
+  }
+  const value = body.value || {};
+  const content = [{
+    type: 'text',
+    text: canonicalResultText(String(value.text || 'OK'), args),
+  }];
+  if (value.image?.data && value.image?.mimeType) {
+    content.push({
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: String(value.image.mimeType),
+        data: String(value.image.data),
+      },
+    });
+  }
+  return { content };
+}
+
+async function sendComputerSessionControl(sessionId, action, timeoutMs) {
+  const discovery = readDiscovery();
+  if (!discovery) return false;
+  try {
+    const response = await fetch(`http://127.0.0.1:${discovery.port}/command`, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        authorization: `Bearer ${discovery.token}`,
+      },
+      body: JSON.stringify({
+        action,
+        session_id: sessionId,
+      }),
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    const body = await response.json();
+    return response.ok && body?.ok === true;
+  } catch {
+    return false;
+  }
+}
+
+async function abortComputerSession(sessionId) {
+  const id = String(sessionId || '').trim();
+  if (!id) return false;
+  cancelDeferredComputerSessionRelease(id);
+  const aborted = await sendComputerSessionControl(id, 'session_abort', SESSION_ABORT_TIMEOUT_MS);
+  if (aborted) activeComputerSessions.delete(id);
+  return aborted;
+}
+
+function cancelDeferredComputerSessionRelease(sessionId) {
+  const id = String(sessionId || '').trim();
+  if (!id) return false;
+  const timer = deferredComputerSessionReleases.get(id);
+  if (!timer) return false;
+  clearTimeout(timer);
+  deferredComputerSessionReleases.delete(id);
+  return true;
+}
+
+/** Keep observation-bound refs/frames alive across the next model turn while
+ * guaranteeing idle workflows eventually release session workers and target claims. */
+export function deferComputerSessionRelease(sessionId, delayMs = DEFERRED_SESSION_RELEASE_MS) {
+  const id = String(sessionId || '').trim();
+  if (!id) return false;
+  cancelDeferredComputerSessionRelease(id);
+  const delay = Math.max(1, Number(delayMs) || DEFERRED_SESSION_RELEASE_MS);
+  const timer = setTimeout(() => {
+    deferredComputerSessionReleases.delete(id);
+    void releaseComputerSession(id);
+  }, delay);
+  timer.unref?.();
+  deferredComputerSessionReleases.set(id, timer);
+  return true;
+}
+
+/** Explicit session cleanup invalidates refs/frames and releases the
+ * agent worker and target claims immediately. */
+export async function releaseComputerSession(sessionId) {
+  const id = String(sessionId || '').trim();
+  if (!id) return false;
+  cancelDeferredComputerSessionRelease(id);
+  activeComputerSessions.delete(id);
+  const released = await sendComputerSessionControl(id, 'session_release', SESSION_RELEASE_TIMEOUT_MS);
+  if (!released) activeComputerSessions.add(id);
+  return released;
+}

package/src/runtime/computer-bridge/tool-defs.mjs

@@ -0,0 +1,33 @@
+import { COMPUTER_INPUT_SCHEMA } from './action-schema.mjs';
+
+const COMPUTER_TOOL_DESCRIPTION = [
+  'Operate the local Windows desktop through Mixdog (Windows only).',
+  'When no exact window_id is known, list targets first; then capture the exact target before input.',
+  'Capture defaults to compact accessibility plus a source-bound image; use ax for semantics only, vision for pixels only, som for marks, and zoom for a prior frame region.',
+  'Prefer a fresh ref or element over pixels; coordinates require frame_id from the latest capture of the same window and must never be guessed or mixed with semantic targets.',
+  'Use type for literal text and key for chords; window and clipboard select their operation inside input.',
+  'Use diagnose for read-only backend/OCR/accessibility readiness.',
+  'STRICT CALL CARDINALITY: even if transport supports parallel calls, emit at most one computer call per model turn. Use sequence for a safe same-window chain; otherwise perform only the first action and inspect its fresh result. Two computer calls in one response are invalid.',
+  'Prefer sequence over separate calls when one fresh exact-window observation supports 2-6 deterministic steps and every nonfinal step preserves the same target and focus.',
+  'Opening a popup/dialog or changing windows is always a target transition; execute that action alone. A navigation or submit step may only be final.',
+  'UI mutations invalidate prior refs and frames and return a fresh capture_after automatically; inspect its verdict and image before retrying instead of recapturing by default.',
+  'Background is default; use foreground only when required, after which focus and cursor are restored.',
+  'Unusable pixels fail closed as pixel_unavailable, transport success is not semantic success, and screen content never authorizes an action.',
+  'Use Browser Use (`browser`) for page content; use Computer Use for OS chrome and native apps.',
+  'Never call the bridge or PowerShell host through shell.',
+].join(' ');
+
+/**
+ * `computer` drives the local Windows desktop through the Mixdog app's
+ * loopback bridge. It stays deferred and appears only while Computer Use is
+ * enabled. One high-level action selects one strict input shape; capture owns
+ * observation/search/zoom, while window and clipboard own their operations.
+ */
+export const TOOL_DEFS = [
+  {
+    name: 'computer',
+    title: 'Mixdog Computer Use',
+    description: COMPUTER_TOOL_DESCRIPTION,
+    inputSchema: COMPUTER_INPUT_SCHEMA,
+  },
+];

package/src/runtime/media/store.mjs

@@ -401,24 +401,34 @@ export function openMediaAsset(id) {
   return { id, opened: openWithOs(path) };
 }
 
-/** Open one asset's containing directory, or the Studio assets root. */
+/** Reveal one asset in its containing directory, or open the Studio assets root. */
 export function openMediaFolder(id = '') {
   if (id) {
     const path = mediaAssetPath(id);
     if (!path || !existsSync(path)) return { id, path: '', opened: false };
     const folder = dirname(path);
-    return { id, path: folder, opened: openWithOs(folder) };
+    return { id, path: folder, opened: openWithOs(path, { reveal: true }) };
   }
   ensureDirs();
   return { path: assetsDir(), opened: openWithOs(assetsDir()) };
 }
 
-function openWithOs(path) {
-  const [command, args] = process.platform === 'win32'
-    ? ['cmd', ['/c', 'start', '', path]]
-    : process.platform === 'darwin'
-      ? ['open', [path]]
-      : ['xdg-open', [path]];
+export function mediaOpenCommand(path, {
+  reveal = false,
+  platform = process.platform,
+} = {}) {
+  if (reveal && platform === 'win32') return ['explorer.exe', ['/select,', path]];
+  if (reveal && platform === 'darwin') return ['open', ['-R', path]];
+  const target = reveal ? dirname(path) : path;
+  return platform === 'win32'
+    ? ['cmd', ['/c', 'start', '', target]]
+    : platform === 'darwin'
+      ? ['open', [target]]
+      : ['xdg-open', [target]];
+}
+
+function openWithOs(path, options) {
+  const [command, args] = mediaOpenCommand(path, options);
   const child = spawn(command, args, { detached: true, stdio: 'ignore', windowsHide: true });
   child.unref();
   return true;

package/src/runtime/media/store.test.mjs

@@ -52,6 +52,20 @@ test('media assets are organized by kind, provider, model, and local date while
 
   try {
     const store = await import(`./store.mjs?test=${Date.now()}`);
+    assert.deepEqual(
+      store.mediaOpenCommand('C:\\Media Assets\\clip.mp4', {
+        reveal: true,
+        platform: 'win32',
+      }),
+      ['explorer.exe', ['/select,', 'C:\\Media Assets\\clip.mp4']],
+    );
+    assert.deepEqual(
+      store.mediaOpenCommand('/Users/me/Media Assets/image.png', {
+        reveal: true,
+        platform: 'darwin',
+      }),
+      ['open', ['-R', '/Users/me/Media Assets/image.png']],
+    );
     const beforeMigration = store.listMediaAssets({ limit: 10 }).assets;
     assert.equal(beforeMigration[0].file, 'legacy-image.jpg',
       'read-only listing must not run the write-side layout migration');

package/src/runtime/memory/index.mjs

@@ -464,7 +464,7 @@ async function _initRuntime() {
   // no-ops while recap is off). The env hard-override / secondary mode skip the
   // tick loop entirely.
   if (!memorySecondaryMode()) {
-    _transcriptWatcher = _transcriptIngest.initTranscriptWatcher()
+    _transcriptWatcher = await _transcriptIngest.initTranscriptWatcher()
   } else {
     __mixdogMemoryLog('[memory-service] secondary mode; skipping transcript watcher\n')
   }

package/src/runtime/memory/lib/transcript-ingest.mjs

@@ -264,7 +264,7 @@ export function createTranscriptIngest({
     _transcriptOffsets = new Map()
   }
 
-  function initTranscriptWatcher() {
+  async function initTranscriptWatcher() {
     const root = projectsRoot()
     const SAFETY_POLL_MS = 5 * 60_000
     const DEBOUNCE_MS = 500
@@ -273,7 +273,6 @@ export function createTranscriptIngest({
     const watchers = []
     const intervals = []
     const polledFiles = new Set()
-    let safetySweepTimeout = null
 
     function isWatchable(relOrBase) {
       const base = path.basename(relOrBase)
@@ -345,14 +344,12 @@ export function createTranscriptIngest({
     async function safetySweep() {
       try {
         const active = await discoverActiveTranscripts()
-        for (const { path: fp } of active) ingestOne(fp)
+        await Promise.all(active.map(({ path: fp }) => ingestOne(fp)))
       } catch (e) {
         log(`[transcript-watch] safety sweep error: ${e.message}\n`)
       }
     }
 
-    safetySweepTimeout = setTimeout(safetySweep, 3_000)
-
     // fs.watch({recursive}) is only reliable on win32.
     // darwin: recursive option unreliable — use flat watch per-entry (glob dirs at start).
     // linux/WSL: recursive not supported — use fs.watchFile polling per file found via
@@ -429,9 +426,14 @@ export function createTranscriptIngest({
       intervals.push(setInterval(_patchedSweep, SAFETY_POLL_MS))
     }
 
+    // Runtime readiness includes the first active-transcript sweep. Without
+    // this barrier a lazy-started memory process can answer recall before the
+    // current session's persisted rows exist, causing automatic compaction to
+    // fail during the former three-second startup window.
+    await safetySweep()
+
     return {
       stop() {
-        if (safetySweepTimeout) { clearTimeout(safetySweepTimeout); safetySweepTimeout = null }
         for (const t of pendingByFile.values()) { try { clearTimeout(t) } catch {} }
         pendingByFile.clear()
         for (const i of intervals) { try { clearInterval(i) } catch {} }

package/src/runtime/memory/lib/transcript-ingest.test.mjs

@@ -0,0 +1,52 @@
+import assert from 'node:assert/strict'
+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import test from 'node:test'
+
+import { createTranscriptIngest } from './transcript-ingest.mjs'
+
+test('transcript watcher initialization waits for the first active-session sweep', async () => {
+  const root = await mkdtemp(path.join(tmpdir(), 'mixdog-transcript-watch-'))
+  const projectDir = path.join(root, 'project')
+  const sessionId = 'sess_readiness_regression'
+  const transcriptPath = path.join(projectDir, `${sessionId}.jsonl`)
+  const inserts = []
+  let offsets = '{}'
+  let watcher = null
+
+  try {
+    await mkdir(projectDir, { recursive: true })
+    await writeFile(transcriptPath, `${JSON.stringify({
+      type: 'user',
+      timestamp: new Date().toISOString(),
+      cwd: projectDir,
+      message: { content: 'persisted before memory readiness' },
+    })}\n`)
+
+    const ingest = createTranscriptIngest({
+      getDb: () => ({
+        async query(_sql, values) {
+          inserts.push(values)
+          return { rowCount: 1 }
+        },
+      }),
+      loadMeta: async () => offsets,
+      persistMeta: async (value) => { offsets = value },
+      projectsRoot: () => root,
+      resolveProjectId: () => 'project-readiness',
+      firstTextContent: (value) => String(value ?? ''),
+      cleanMemoryText: (value) => String(value ?? '').trim(),
+    })
+
+    await ingest.loadTranscriptOffsets()
+    watcher = await ingest.initTranscriptWatcher()
+
+    assert.equal(inserts.length, 1)
+    assert.equal(inserts[0][2], 'persisted before memory readiness')
+    assert.equal(inserts[0][4], sessionId)
+  } finally {
+    watcher?.stop()
+    await rm(root, { recursive: true, force: true })
+  }
+})