mixdog 1.3.5 → 1.4.1

package/src/runtime/agent/orchestrator/session/catalog-db/schema.mjs

@@ -11,7 +11,7 @@ import { createHash } from 'node:crypto';
 
 /** Bumped whenever the table layout or a codec changes: the DB is then
  *  dropped and backfilled again from the JSON catalog. */
-export const SCHEMA_VERSION = 10;
+export const SCHEMA_VERSION = 11;
 /** PRAGMA application_id ('MXSC'): a file without it is not this catalog. */
 export const APPLICATION_ID = 0x4d585343;
 
@@ -68,6 +68,9 @@ const FIELDS = [
   // Places of the incoming messages with text among the counted ones, for the
   // unread count of a bot home / room log; absent otherwise (schema 10).
   ['incomingTail', 'incoming_tail', 'optjson'],
+  // Compactions the session has had: the epoch of its message numbering;
+  // absent for a row written before schema 11.
+  ['compactCount', 'compact_count', 'optjson'],
 ];
 
 const COLUMN_TYPES = { text: 'TEXT', real: 'REAL', int: 'INTEGER', bool: 'INTEGER', json: 'TEXT', optjson: 'TEXT' };

package/src/runtime/agent/orchestrator/session/compact/execution-tail.mjs

@@ -9,6 +9,7 @@ import {
   isProtectedContextAckMessage,
   latestActualUserInstructionIndex,
   pendingTaskNotificationIndexes,
+  requestStart,
 } from './messages.mjs';
 import { staleObservations, supersedeObservation } from './observation-supersession.mjs';
 import { carryMessageId, withoutMessageIdsReplacer } from '../message-ids.mjs';
@@ -101,14 +102,6 @@ export function executionTokens(messages) {
   );
 }
 
-function requestStart(messages, index) {
-  for (let i = index; i >= 0; i -= 1) {
-    if (!isActualUserInstructionMessage(messages[i])) continue;
-    if (messages[i]?.meta?.source !== 'steering') return i;
-  }
-  return latestActualUserInstructionIndex(messages.slice(0, index + 1));
-}
-
 // Each assistant tool-call turn with the tool results that directly follow it.
 function executionGroups(messages) {
   const groups = [];

package/src/runtime/agent/orchestrator/session/compact/messages.mjs

@@ -43,6 +43,29 @@ export function latestActualUserInstructionIndex(messages) {
   return -1;
 }
 
+// The user instruction that began the request `index` belongs to; a steering
+// message continues the request instead of starting one.
+export function requestStart(messages, index) {
+  for (let i = index; i >= 0; i -= 1) {
+    if (!isActualUserInstructionMessage(messages[i])) continue;
+    if (messages[i]?.meta?.source !== 'steering') return i;
+  }
+  return latestActualUserInstructionIndex(messages.slice(0, index + 1));
+}
+
+// A tool call's arguments as an object, whether stored parsed or as JSON text.
+export function callArguments(call) {
+  let args = call?.arguments;
+  if (typeof args === 'string') {
+    try {
+      args = JSON.parse(args);
+    } catch {
+      return {};
+    }
+  }
+  return args && typeof args === 'object' ? args : {};
+}
+
 export function latestActualUserInstructionMessage(messages) {
   const index = latestActualUserInstructionIndex(messages);
   return index >= 0 ? { ...messages[index] } : null;

package/src/runtime/agent/orchestrator/session/compact/observation-supersession.mjs

@@ -1,9 +1,12 @@
-// Observations that a later one of the same browser page or desktop window
-// replaced. Such a result's refs have expired, and its element list and pixels
-// describe a state that no longer exists. Only compaction drops them: between
-// compactions the transcript stays append-only so the provider cache prefix
-// holds, and compaction rewrites that prefix anyway. The execution archive
-// keeps every original.
+// Observations that a later one of the same browser page, desktop window,
+// visualize tab (inspect / check) or image file replaced. Such a result's refs
+// have expired, and its element list and pixels describe a state that no
+// longer exists. Only compaction drops them: between compactions the
+// transcript stays append-only so the provider cache prefix holds, and
+// compaction rewrites that prefix anyway. The execution archive keeps every
+// original.
+import { callArguments, requestStart } from './messages.mjs';
+
 const BROWSER_TOOLS = new Set(['browser', 'browser_devtools']);
 const SNAPSHOT_LINE = /(?:^|\n)Snapshot:\s+((p\d+)-s\d+)\b/;
 const TERMINAL_LINE = /(?:^|\n)Terminal:\s+(session-terminal:\S+)/;
@@ -52,13 +55,40 @@ function containsRun(newer, older) {
   return false;
 }
 
-// What one tool result observed, keyed by the page or window it describes.
-function observationOf(message, toolName) {
+const VISUALIZE_READS = new Set(['inspect', 'check']);
+const IMAGE_FILE = /\.(?:png|jpe?g|gif|webp)$/i;
+// What a superseded computer frame keeps besides its reference.
+const FRAME_OUTCOME_KEYS = ['ok', 'action', 'actions', 'status', 'error', 'window_id'];
+
+// A visualize read-out is the same observation only for the same tab, kind of
+// read, viewport and scope: a mobile inspect does not replace a desktop one.
+function visualizeKey(args) {
+  const input = args.input && typeof args.input === 'object' ? args.input : {};
+  if (!VISUALIZE_READS.has(args.action) || typeof input.tab !== 'string' || !input.tab) return null;
+  const scope = args.action === 'inspect' ? `:${JSON.stringify(input.viewport ?? null)}:${input.selector ?? ''}` : '';
+  return { key: `visualize:${args.action}:${input.tab}${scope}`, action: args.action, tab: input.tab };
+}
+
+// What one tool result observed, keyed by the page, window, tab or file it
+// describes.
+function observationOf(message, call) {
   if (message?.role !== 'tool' || message.toolKind === 'error') return null;
+  const toolName = String(call?.name || '');
   const parts = contentParts(message.content);
+  if (toolName === 'read') {
+    const args = callArguments(call);
+    const path = typeof args.path === 'string' ? args.path : args.file_path;
+    if (typeof path !== 'string' || !IMAGE_FILE.test(path) || !parts.some(isImagePart)) return null;
+    return { key: `image:${path.replace(/\\/g, '/')}`, imagePath: path, requestScoped: true };
+  }
   const textIndex = parts.findIndex((part) => part?.type === 'text' && !isImagePart(part));
   if (textIndex < 0) return null;
   const text = String(parts[textIndex].text);
+  if (toolName === 'visualize') {
+    if (/^\[Visualize \w+ of tab \S+ superseded:/.test(text)) return null;
+    const read = visualizeKey(callArguments(call));
+    return read ? { key: read.key, parts, textIndex, visualize: read, requestScoped: true } : null;
+  }
   if (BROWSER_TOOLS.has(toolName)) {
     const snapshot = SNAPSHOT_LINE.exec(text);
     return snapshot ? { key: `browser:${snapshot[2]}`, parts, textIndex, text, snapshot } : null;
@@ -82,40 +112,90 @@ function observationOf(message, toolName) {
   const frame = value?.observation && typeof value.observation === 'object' ? value.observation : value;
   const windowId = frame?.window_id || value?.window_id;
   const observed = Array.isArray(frame?.elements) || frame?.ocr !== undefined || parts.some(isImagePart);
-  return windowId && observed ? { key: `computer:${windowId}`, parts, textIndex, value, frame } : null;
+  if (windowId && observed) return { key: `computer:${windowId}`, parts, textIndex, value, frame };
+  // A stub an earlier compaction wrote in the fuller format.
+  const slim =
+    typeof frame?.superseded === 'string' &&
+    Object.keys(frame).some((name) => name !== 'superseded' && !FRAME_OUTCOME_KEYS.includes(name));
+  return slim ? { stub: true, parts, textIndex, value, frame, reference: frame.superseded } : null;
 }
 
-/** Tool results that a later observation of the same page or window
- *  replaced, keyed by transcript index. */
+// Transcript indices whose results are still in use and must stay verbatim:
+// the latest tool-call group (the batch the session continues from, which
+// compaction always retains) and any group with a call that has no result yet.
+function protectedObservationIndices(messages) {
+  const protectedIndices = new Set();
+  const protect = (from, to) => {
+    for (let index = from; index < to; index += 1) protectedIndices.add(index);
+  };
+  let latest = null;
+  for (let index = 0; index < messages.length; index += 1) {
+    const message = messages[index];
+    if (message?.role !== 'assistant' || !message.toolCalls?.length) continue;
+    const start = index;
+    const answered = new Set();
+    while (messages[index + 1]?.role === 'tool') {
+      index += 1;
+      answered.add(messages[index].toolCallId);
+    }
+    latest = [start, index + 1];
+    if (message.toolCalls.some((call) => !answered.has(call?.id))) protect(...latest);
+  }
+  if (latest) protect(...latest);
+  return protectedIndices;
+}
+
+/** Tool results that a later observation of the same page, window, tab or
+ *  file replaced, keyed by transcript index. Protected results are never
+ *  listed, though they still count as the newer observation of what they
+ *  describe. An image file read or visualize read-out is also kept for the
+ *  whole active request (everything since the last user instruction), since
+ *  the request may compare it with a later one; once a newer user message
+ *  exists it is replaceable. */
 export function staleObservations(messages) {
-  const toolNames = new Map();
+  const protectedIndices = protectedObservationIndices(messages);
+  const activeRequest = Math.max(requestStart(messages, messages.length - 1), 0);
+  const calls = new Map();
   for (const message of messages) {
     if (message?.role !== 'assistant' || !Array.isArray(message.toolCalls)) continue;
-    for (const call of message.toolCalls) if (call?.id) toolNames.set(call.id, String(call.name || ''));
+    for (const call of message.toolCalls) if (call?.id) calls.set(call.id, call);
   }
   const newest = new Set();
   const terminalReads = new Map();
   const stale = new Map();
   for (let index = messages.length - 1; index >= 0; index -= 1) {
-    const found = observationOf(messages[index], toolNames.get(messages[index]?.toolCallId));
+    const found = observationOf(messages[index], calls.get(messages[index]?.toolCallId));
     if (!found) continue;
-    if (found.body) {
+    const replaceable = !protectedIndices.has(index) && !(found.requestScoped && index >= activeRequest);
+    if (found.stub) {
+      if (replaceable) stale.set(index, found);
+    } else if (found.body) {
       const newer = terminalReads.get(found.key) ?? [];
-      if (newer.some((body) => containsRun(body, found.body))) stale.set(index, found);
+      if (replaceable && newer.some((body) => containsRun(body, found.body))) stale.set(index, found);
       newer.push(found.body);
       terminalReads.set(found.key, newer);
-    } else if (newest.has(found.key)) stale.set(index, found);
-    else newest.add(found.key);
+    } else if (newest.has(found.key)) {
+      if (replaceable) stale.set(index, found);
+    } else newest.add(found.key);
   }
   return stale;
 }
 
 /** The result without its replaced observation. A browser result keeps what
  *  precedes its snapshot (effect notes, a script's value) and a computer
- *  result its action outcome; neither keeps elements, OCR text or pixels. */
+ *  result its action outcome plus a reference; none keeps elements, OCR text,
+ *  frame metadata or pixels. A visualize read-out or image file read becomes a
+ *  one-line placeholder naming what it was and how to get it again. */
 export function supersedeObservation(message, found, origin) {
   let text;
-  if (found.terminal) {
+  if (found.imagePath) {
+    text = `[Image read of ${found.imagePath} superseded: a newer read of this file follows. Read the file again to view it. ${origin}]`;
+    return { ...message, content: withParts(message.content, [{ type: 'text', text }]) };
+  }
+  if (found.visualize) {
+    const { action, tab } = found.visualize;
+    text = `[Visualize ${action} of tab ${tab} superseded: a newer ${action} of this tab follows. Run visualize ${action} on the tab again to get it. ${origin}]`;
+  } else if (found.terminal) {
     text = `[Terminal ${found.terminal} read superseded: a newer read of this tab follows. ${origin}]`;
   } else if (found.snapshot) {
     const { snapshot } = found;
@@ -126,9 +206,9 @@ export function supersedeObservation(message, found, origin) {
     const note = `[Snapshot ${snapshot[1]} superseded: a newer snapshot of this page follows. ${origin}]`;
     text = before ? `${before}\n\n${note}` : note;
   } else {
-    const frame = { ...found.frame, superseded: `a newer observation of this window follows. ${origin}` };
-    delete frame.elements;
-    delete frame.ocr;
+    const frame = {};
+    for (const name of FRAME_OUTCOME_KEYS) if (found.frame[name] !== undefined) frame[name] = found.frame[name];
+    frame.superseded = found.reference ?? `a newer observation of this window follows. ${origin}`;
     text = JSON.stringify(found.frame === found.value ? frame : { ...found.value, observation: frame });
   }
   const parts = found.parts.flatMap((part, index) => {