mixdog 1.3.5 → 1.4.1

package/src/runtime/agent/orchestrator/session/compact/runner.mjs

@@ -1,6 +1,7 @@
 // Rule-based Compact preserves conversation verbatim by default. A bounded
 // cumulative handoff replaces only the conversation part when requested by
 // its independent threshold policy.
+import { createHash } from 'node:crypto';
 import { estimateMessagesTokens, reconcileDedupStubs, sanitizeToolPairs } from '../context-utils.mjs';
 import { HANDOFF_TIMEOUT_MAX_MS, SUMMARY_OUTPUT_TOKENS, compactDebugLog } from './constants.mjs';
 import {
@@ -11,6 +12,7 @@ import {
   textByteLength,
 } from './text-utils.mjs';
 import {
+  callArguments,
   contextAckMessage,
   isCompactConversationRow,
   isProtectedContextAckMessage,
@@ -22,8 +24,9 @@ import {
 } from './messages.mjs';
 import { activeTurnContinuationMessage } from './continuation.mjs';
 import { EXECUTION_RECOVERY_SOURCE, buildExecutionTail } from './execution-tail.mjs';
-import { latestSkillBodies } from '../../context/skill-state.mjs';
+import { latestSkillBodies, skillMessageText } from '../../context/skill-state.mjs';
 import { effectiveBudget } from './budget.mjs';
+import { parseMcpToolName } from '../../../../shared/tool-primitives.mjs';
 import { stripRuntimeUserContext, withRuntimeUserContext } from '../runtime-user-context.mjs';
 import { carryMessageId, messageIdOf } from '../message-ids.mjs';
 import { normalizeIngestRole, shouldExcludeIngestMessage } from '../../../../memory/lib/session-ingest.mjs';
@@ -69,12 +72,23 @@ function splitFreshSource(messages) {
       break;
     }
   }
+  // Where the previous Compact's skill-use window opens: `live` index after
+  // the history it kept (see SKILL_WINDOW_SOURCE). Without a marker the whole
+  // transcript is the window.
+  const isKept = (message) => !isSummaryMessage(message) && message?.meta?.source !== SKILL_WINDOW_SOURCE;
+  const markerAt = conversation.findLastIndex((message) => message?.meta?.source === SKILL_WINDOW_SOURCE);
+  const span = conversation[markerAt]?.meta?.span;
+  const skillWindowStart =
+    markerAt >= 0 && Number.isInteger(span) && span >= 0
+      ? conversation.slice(0, markerAt).filter(isKept).length + span
+      : 0;
   const latestUserIndex = latestActualUserInstructionIndex(conversation);
   return {
     protectedPrefix,
     previousSummary: previousSummaryMessage?.content || null,
     previousSummaryMessage,
     previousAck,
+    skillWindowStart,
     // Budgeting, AI input and rule-only retention use this same projection.
     // Keep current-turn reminders; remove only proven runtime additions from
     // older requests. Unmarked legacy text remains part of the summary budget.
@@ -85,7 +99,7 @@ function splitFreshSource(messages) {
       .map((message, index) =>
         index === latestUserIndex ? message : referenceMessageMedia(stripRuntimeUserContext(message))
       )
-      .filter((message) => !isSummaryMessage(message)),
+      .filter(isKept),
     sanitized,
   };
 }
@@ -134,9 +148,11 @@ function toolCallSummaryLine(call, result) {
   // Arguments arrive redacted (splitFreshSource); the outcome is redacted
   // here, before any cut can split a secret from its key.
   const outcome = redactRawSecretString(contentText(result?.content)).trim();
-  // An ask_user result is the user's own answer: never cut.
+  // An ask_user or handoff_user result is the user's own answer: never cut.
   const shown =
-    call.name === 'ask_user' ? outcome : boundedSummaryText(outcome.replace(/\s+/g, ' '), SUMMARY_TOOL_OUTCOME_CHARS);
+    call.name === 'ask_user' || call.name === 'handoff_user'
+      ? outcome
+      : boundedSummaryText(outcome.replace(/\s+/g, ' '), SUMMARY_TOOL_OUTCOME_CHARS);
   return `[tool call] ${call.name}(${args}) → ${result ? shown || '(empty result)' : '(no result)'}`;
 }
 
@@ -413,6 +429,47 @@ function prependLatestUserContext(message, prefix) {
   });
 }
 
+const LOADED_SKILL_RESULT_RE = /^Loaded (?:built-in )?skill: /;
+
+// A Skill load the retained execution tail keeps states what the skill
+// delivered then: its tool list (the native payload or the summary text). It
+// is brought to what the skill delivers now, in place, so its call stays
+// paired: a skill that no longer loads keeps a result saying so, with no tool
+// payload; a skill whose dependencies changed carries its current payload.
+// Failed loads (errors) are left as they are. `currentSkill` absent, or a
+// skill's `native` undefined (a preview), keeps the result.
+function reconcileRetainedSkillLoads(messages, currentSkill) {
+  if (!currentSkill) return messages;
+  const calls = new Map(
+    messages.flatMap((message) =>
+      message?.role === 'assistant' ? (message.toolCalls || []).map((call) => [call.id, call]) : []
+    )
+  );
+  const withoutPayload = ({ nativeToolSearch: _native, ...message }, content) => ({ ...message, content });
+  return messages.map((message) => {
+    if (message?.role !== 'tool') return message;
+    const call = calls.get(message.toolCallId);
+    const name = call?.name === 'Skill' ? String(call.arguments?.name || '').trim() : '';
+    if (!name) return message;
+    const text = contentText(message.content);
+    if (!message.nativeToolSearch && !LOADED_SKILL_RESULT_RE.test(text)) return message;
+    const skill = currentSkill(name);
+    if (skill === null) {
+      return withoutPayload(
+        message,
+        `Skill "${name}" is no longer available in this session (switched off, removed, or outside this project's scope); its instructions and tools are not active.`
+      );
+    }
+    if (skill?.native === undefined) return message;
+    if (skill.native) {
+      return nativeToolsKey(message.nativeToolSearch) === nativeToolsKey(skill.native)
+        ? message
+        : { ...message, content: skill.native.summary, nativeToolSearch: skill.native };
+    }
+    return !message.nativeToolSearch && text === skill.summary ? message : withoutPayload(message, skill.summary);
+  });
+}
+
 // The retained execution tail plus the current-turn messages that must
 // survive every compaction verbatim.
 function freshContextTail(source, budgetTokens, preserveConversation, opts) {
@@ -423,10 +480,11 @@ function freshContextTail(source, budgetTokens, preserveConversation, opts) {
     preview: opts.preview === true,
   });
   const latestUser = latestActualUserInstructionMessage(source.live);
-  const latestIndex = execution.messages.findLastIndex(
+  const executed = reconcileRetainedSkillLoads(execution.messages, opts.currentSkill);
+  const latestIndex = executed.findLastIndex(
     (message) => message.role === 'user' && message.content === latestUser?.content
   );
-  const retainedTail = execution.messages.map((message, index) =>
+  const retainedTail = executed.map((message, index) =>
     index === latestIndex ? prependLatestUserContext(message, opts.latestUserPrefix) : message
   );
   // The turn's input: the latest real request or an unanswered notification
@@ -459,11 +517,9 @@ function freshContextTail(source, budgetTokens, preserveConversation, opts) {
   };
 }
 
-// Native tool search exposes a skill's linked tools only through the executed
-// Skill result that referenced them. A restored body keeps that pair, unless
-// the retained tail already carries it, so its instructions never return
-// without the tools they require.
-function skillLoaderPair(live, bodyIndex, name, retainedCallIds) {
+// The executed Skill call (its owner message, the call and its result) that
+// delivered the body at `bodyIndex`, or null.
+function skillLoader(live, bodyIndex, name) {
   for (let index = bodyIndex - 1; index >= 0; index -= 1) {
     const result = live[index];
     if (result?.role !== 'tool') continue;
@@ -473,65 +529,211 @@ function skillLoaderPair(live, bodyIndex, name, retainedCallIds) {
         (message) => message?.role === 'assistant' && message.toolCalls?.some((c) => c.id === result.toolCallId)
       );
     const call = owner?.toolCalls.find((entry) => entry.id === result.toolCallId);
-    if (call?.name !== 'Skill' || call.arguments?.name !== name) continue;
-    if (!result.nativeToolSearch || retainedCallIds.has(result.toolCallId)) return [];
-    // The re-attached call is part of its owner message: it keeps that id.
-    return [carryMessageId({ role: 'assistant', content: '', toolCalls: [call] }, owner), result];
+    if (call?.name === 'Skill' && call.arguments?.name === name) return { owner, call, result };
   }
-  return [];
+  return null;
+}
+
+const nativeToolsKey = (native) =>
+  JSON.stringify([native?.provider || '', native?.toolReferences || [], native?.openaiTools || []]);
+
+const SKILL_RESTORE_SOURCE = 'compact-skill-restore';
+
+// Native tool search exposes a skill's linked tools only through the executed
+// Skill result that referenced them. A restored body keeps that pair, unless
+// the retained tail already carries it, so its instructions never return
+// without the tools they require. `current.native` is the payload the skill's
+// current dependencies produce (null: none to expose); the transcript pair is
+// kept while it matches, otherwise rebuilt from it. `current` absent, or its
+// `native` undefined, keeps the transcript pair as it is.
+function skillLoaderPair(live, bodyIndex, name, retainedCallIds, current) {
+  const loader = skillLoader(live, bodyIndex, name);
+  if (loader && retainedCallIds.has(loader.result.toolCallId)) return [];
+  const native = current?.native === undefined ? loader?.result.nativeToolSearch : current.native;
+  if (!native) return [];
+  // The re-attached call is part of its owner message: it keeps that id.
+  const callMessage = (call) =>
+    loader
+      ? carryMessageId({ role: 'assistant', content: '', toolCalls: [call] }, loader.owner)
+      : { role: 'assistant', content: '', toolCalls: [call], meta: { source: SKILL_RESTORE_SOURCE, synthetic: true } };
+  if (loader?.result.nativeToolSearch && nativeToolsKey(loader.result.nativeToolSearch) === nativeToolsKey(native)) {
+    return [callMessage(loader.call), loader.result];
+  }
+  // A body that arrived without a native pair still needs one now: a
+  // per-skill call id, stable across later Compacts that keep it.
+  const call = loader?.call || {
+    id: `skill-restore-${createHash('sha256').update(name).digest('hex').slice(0, 24)}`,
+    name: 'Skill',
+    arguments: { name },
+  };
+  const result = { role: 'tool', toolCallId: call.id, content: native.summary, nativeToolSearch: native };
+  return [callMessage(call), loader ? carryMessageId(result, loader.result) : result];
+}
+
+// The body to restore: the transcript copy while it is what the Skill tool
+// delivers now, else the current body.
+function currentSkillBodyMessage(message, current) {
+  if (!current || skillMessageText(message.content).trimStart() === current.body) return message;
+  return { role: 'user', content: current.body, meta: 'skill' };
 }
 
 const SKILL_RELOAD_SOURCE = 'compact-skill-reload';
 
-function skillReloadReminder(name) {
+// How many Compacts in a row a skill may go unused before even its reload
+// line is dropped: a skill untouched that long belongs to finished work, and
+// the model can still load it by name from the skill manifest.
+export const SKILL_RELOAD_MAX_IDLE = 3;
+
+const SKILL_TOOL_NAMES = new Set(['Skill', 'skill_view']);
+
+function skillReloadReminder(name, idle = 0) {
   return {
     role: 'user',
-    content: `[mixdog-runtime] Skill "${name}" was active, but its instructions did not fit after Compact. Reload it with the Skill tool before relying on them.`,
-    meta: { source: SKILL_RELOAD_SOURCE, synthetic: true, skill: name },
+    content: `[mixdog-runtime] Skill "${name}" was active earlier, but Compact did not keep its instructions. Reload it with the Skill tool before relying on them.`,
+    meta: { source: SKILL_RELOAD_SOURCE, synthetic: true, skill: name, idle },
   };
 }
 
-// Preserve complete, most-recent skill bodies from the durable transcript.
+// Every Compact leaves this marker just ahead of the skills it restores, with
+// `span`: how many messages follow it (restored skills and retained tail).
+// All of them happened before that Compact, so the next one reads the skill
+// use window from after them, whether or not a summary exists. The marker is
+// not conversation: the next Compact removes it and writes a new one.
+const SKILL_WINDOW_SOURCE = 'compact-skill-window';
+
+function skillWindowMarker(span = 0) {
+  return {
+    role: 'user',
+    content: '[mixdog-runtime] Compact boundary.',
+    meta: { source: SKILL_WINDOW_SOURCE, synthetic: true, span },
+  };
+}
+
+// Restore the skills the session used since the previous Compact, each as the
+// Skill tool would deliver it now. A skill is used when it was loaded or one of
+// its linked tools ran in that window (`windowStart` onward); its linked
+// tools are the native tool references of its loader pair and, when the
+// resolver gives them, `tools`. `currentSkill(name)` (the caller's resolver)
+// answers `{ body, native, summary, tools? }` for a skill that still loads in
+// the session, or null for one switched off, outside its scope or removed:
+// such a skill is restored neither as a body nor as a reload line, since its
+// instructions are no longer in force. Without a resolver the transcript
+// copies are kept.
 // Never summarize/truncate operating instructions or displace the handoff
-// and current task. An omitted body leaves a one-line reload reminder, which
-// stands through later Compacts until the skill is loaded again. Reminders
-// are charged to the same budget: every skill's reminder is reserved before
-// any body is restored, and restoring a body releases its own reminder.
-function restoreSkillBodies(live, skillBudget, retainedTail) {
+// and current task. Every other active skill, and a used one whose body does
+// not fit, is left as a one-line reload line. The line counts the Compacts
+// the skill went unused (`meta.idle`) and is dropped past SKILL_RELOAD_MAX_IDLE.
+// Reload lines are charged to the same budget: every possible line is
+// reserved before any body is restored, and restoring a body releases its own.
+function restoreSkillBodies(live, skillBudget, retainedTail, currentSkill = null, windowStart = 0) {
   const retainedCallIds = new Set(retainedTail.filter((m) => m?.role === 'tool').map((m) => m.toolCallId));
-  const bodies = latestSkillBodies(live);
-  const loaded = new Set(bodies.map((body) => body.name));
-  const reload = live
-    .filter((m) => m?.meta?.source === SKILL_RELOAD_SOURCE && !loaded.has(m.meta.skill))
-    .map((m) => m.meta.skill);
-  const reminderCost = (name) => estimateMessagesTokens([skillReloadReminder(name)]);
-  let reserved = [...new Set([...reload, ...loaded])].reduce((sum, name) => sum + reminderCost(name), 0);
-  const dropped = [];
+  const current = (name) => (currentSkill ? currentSkill(name) : undefined);
+  const loads = (name) => current(name) !== null;
+  const transcriptBodies = latestSkillBodies(live);
+  const loaded = new Set(transcriptBodies.map((body) => body.name));
+  const bodies = transcriptBodies.filter((body) => loads(body.name));
+  // Reload lines from earlier Compacts: skill -> Compacts it went unused.
+  const earlier = new Map();
+  for (const m of live) {
+    if (m?.meta?.source !== SKILL_RELOAD_SOURCE || loaded.has(m.meta.skill) || !loads(m.meta.skill)) continue;
+    earlier.delete(m.meta.skill);
+    earlier.set(m.meta.skill, Number.isFinite(m.meta.idle) ? m.meta.idle : 0);
+  }
+
+  // A loader pair can sit anywhere in the transcript (a restored body stays
+  // ahead of the retained tail's pair), so read them all.
+  const results = new Map(live.filter((m) => m?.role === 'tool').map((m) => [m.toolCallId, m]));
+  const loaderTools = new Map();
+  for (const message of live) {
+    for (const call of message?.role === 'assistant' ? message.toolCalls || [] : []) {
+      const references = results.get(call.id)?.nativeToolSearch?.toolReferences;
+      if (call.name !== 'Skill' || !references) continue;
+      const name = String(callArguments(call).name || '').trim();
+      loaderTools.set(name, [...(loaderTools.get(name) || []), ...references]);
+    }
+  }
+  const skillsOfTool = new Map();
+  for (const name of new Set([...earlier.keys(), ...bodies.map((body) => body.name)])) {
+    const skill = current(name);
+    const tools = [...(skill?.native?.toolReferences || []), ...(skill?.tools || []), ...(loaderTools.get(name) || [])];
+    for (const tool of tools) skillsOfTool.set(tool, (skillsOfTool.get(tool) || new Set()).add(name));
+  }
+  // Use since the previous Compact (from `windowStart`): the newest transcript
+  // position of a load or of a linked tool's call.
+  const lastUse = new Map();
+  const noteUse = (name, index) => lastUse.set(name, Math.max(lastUse.get(name) ?? -1, index));
+  live.forEach((message, index) => {
+    if (index < windowStart || message?.role !== 'assistant') return;
+    for (const call of message.toolCalls || []) {
+      const args = callArguments(call);
+      if (SKILL_TOOL_NAMES.has(call.name)) noteUse(String(args.name || '').trim(), index);
+      else {
+        const tool = call.name === 'call_tool' ? args.name : call.name;
+        const mcpServer = `mcp:${parseMcpToolName(tool)?.server}`;
+        for (const name of [...(skillsOfTool.get(tool) || []), ...(skillsOfTool.get(mcpServer) || [])]) {
+          noteUse(name, index);
+        }
+      }
+    }
+  });
+  for (const { name, message } of bodies) {
+    const index = live.lastIndexOf(message);
+    if (index >= windowStart) noteUse(name, index);
+  }
+
+  // Every active skill without a body ends as a reload line: an unused one
+  // ages by a Compact and goes once past the bound; one used (or whose body
+  // did not fit) starts over.
+  const reminders = new Map();
+  for (const [name, idle] of earlier) {
+    if (lastUse.has(name)) reminders.set(name, 0);
+    else if (idle < SKILL_RELOAD_MAX_IDLE) reminders.set(name, idle + 1);
+  }
+  const used = bodies.filter(({ name }) => lastUse.has(name)).sort((a, b) => lastUse.get(b.name) - lastUse.get(a.name));
+  const unused = bodies.filter(({ name }) => !lastUse.has(name));
+
+  const reminderCost = (name, idle = 0) => estimateMessagesTokens([skillReloadReminder(name, idle)]);
+  let reserved =
+    [...reminders].reduce((sum, [name, idle]) => sum + reminderCost(name, idle), 0) +
+    unused.reduce((sum, { name }) => sum + reminderCost(name, 1), 0) +
+    used.reduce((sum, { name }) => sum + reminderCost(name), 0);
+  const dropped = new Set();
   const restoredSkills = [];
   let skillTokens = 0;
-  for (const { name, message } of bodies.reverse()) {
-    const restored = [...skillLoaderPair(live, live.lastIndexOf(message), name, retainedCallIds), message];
+  for (const { name, message } of used) {
+    const skill = current(name);
+    const restored = [
+      ...skillLoaderPair(live, live.lastIndexOf(message), name, retainedCallIds, skill),
+      currentSkillBodyMessage(message, skill),
+    ];
     const cost = estimateMessagesTokens(restored);
     const own = reminderCost(name);
     if (skillTokens + cost + reserved - own > skillBudget) {
-      dropped.unshift(name);
+      dropped.add(name);
       continue;
     }
     restoredSkills.unshift(...restored);
     skillTokens += cost;
     reserved -= own;
   }
-  // Newest skill first; only a budget too small for the reminders themselves
+  // In transcript order, after the lines of earlier Compacts.
+  for (const { name } of bodies) {
+    if (!lastUse.has(name)) reminders.set(name, 1);
+    else if (dropped.has(name)) reminders.set(name, 0);
+  }
+  // Newest skill first; only a budget too small for the lines themselves
   // leaves one out.
-  const pending = [...new Set([...reload, ...dropped])];
+  const pending = [...reminders];
   const reminded = new Set();
-  for (const name of pending.toReversed()) {
-    const cost = reminderCost(name);
+  for (const [name, idle] of pending.toReversed()) {
+    const cost = reminderCost(name, idle);
     if (skillTokens + cost > skillBudget) continue;
     reminded.add(name);
     skillTokens += cost;
   }
-  restoredSkills.push(...pending.filter((name) => reminded.has(name)).map(skillReloadReminder));
+  restoredSkills.push(
+    ...pending.filter(([name]) => reminded.has(name)).map(([name, idle]) => skillReloadReminder(name, idle))
+  );
   return { restoredSkills, skillTokens };
 }
 
@@ -716,7 +918,19 @@ function assertHandoffFits(summaryMessage, room) {
   }
 }
 
-export function freshContextCompactMessages(messages, budgetTokens, opts = {}) {
+// One resolution per skill per Compact: the retained tail and the restored
+// bodies read the same answer (the real pass re-arms dependencies once).
+function memoizedSkillResolver(resolve) {
+  if (!resolve) return null;
+  const known = new Map();
+  return (name) => {
+    if (!known.has(name)) known.set(name, resolve(name));
+    return known.get(name);
+  };
+}
+
+export function freshContextCompactMessages(messages, budgetTokens, options = {}) {
+  const opts = { ...options, currentSkill: memoizedSkillResolver(options.currentSkill) };
   const startedAt = Date.now();
   const targetBudget = effectiveBudget(budgetTokens, opts);
   const baseSanitized = reconcileDedupStubs(sanitizeToolPairs(messages));
@@ -756,12 +970,33 @@ export function freshContextCompactMessages(messages, budgetTokens, opts = {}) {
   const summaryContent = String(summaryMessage?.content || '');
   assertHandoffFits(summaryMessage, handoff.room);
   const skillBudget = freshSkillBudget(targetBudget, budget, [...stableHead, ...tail.volatileTail]);
-  const skills = { ...restoreSkillBodies(source.live, skillBudget, tail.volatileTail), skillBudget };
-  const stablePrefixMessages = [...stableHead, ...skills.restoredSkills];
-  const result = rebaseCompactedEffortConfiguration(
+  // Restored skills carry the window marker, which is charged to their budget.
+  const markerCost = estimateMessagesTokens([skillWindowMarker()]);
+  const skills = {
+    ...restoreSkillBodies(
+      source.live,
+      Math.max(0, skillBudget - markerCost),
+      tail.volatileTail,
+      opts.currentSkill,
+      source.skillWindowStart
+    ),
+    skillBudget,
+  };
+  // A session with no skill state has no use to window: no marker.
+  const marker = skills.restoredSkills.length ? [skillWindowMarker()] : [];
+  const stablePrefixMessages = [...stableHead, ...marker, ...skills.restoredSkills];
+  const compacted = rebaseCompactedEffortConfiguration(
     baseSanitized,
     reconcileDedupStubs(sanitizeToolPairs([...stablePrefixMessages, ...tail.volatileTail]))
   );
+  // The next Compact reads everything after the marker as history, not as use.
+  const markerAt = compacted.findLastIndex((message) => message?.meta?.source === SKILL_WINDOW_SOURCE);
+  const result =
+    markerAt < 0
+      ? compacted
+      : compacted.map((message, index) =>
+          index === markerAt ? skillWindowMarker(compacted.length - markerAt - 1) : message
+        );
   const finalTokens = estimateMessagesTokens(result);
   if (finalTokens > budget) {
     throw new Error(`freshContextCompactMessages: compacted result exceeds budget=${budget} (result=${finalTokens})`);