@modelprofile.com/flexharness 10.13.0 → 10.15.0

package/ts_compaction/smartagent.compaction.ts

@@ -25,6 +25,9 @@ When constructing the summary, try to stick to this template:
 /** The instructions of a summary call whose caller names none of the conversation's own. */
 const COMPACTION_INSTRUCTIONS = 'You summarize a conversation so that another agent can continue its work.';
 
+/** Added to the summary request when it carries the conversation's tools. */
+const NO_TOOL_CALLS = 'Do not call any tool. Reply with the summary text only.';
+
 export interface ICompactMessagesOptions {
   abortSignal?: AbortSignal;
   /**
@@ -45,13 +48,51 @@ export interface ICompactMessagesOptions {
    * options' `providerOptions` through.
    */
   providerOptions?: plugins.ProviderOptions;
+  /**
+   * The tools the conversation's turns are sent with. The summary call carries their definitions,
+   * none of their code, so its request repeats the turns' prefix and reads it from the provider's
+   * prompt cache, unless the definitions outweigh the history nine times, when reading them would
+   * cost more than the history saves. A context compactor passes its options' `tools` through.
+   */
+  tools?: plugins.ToolSet;
+}
+
+/**
+ * Whether the summary request should carry the turns' tool definitions. A cached prefix is read at
+ * about a tenth of the input price (Anthropic and OpenAI), so repeating the turns' prefix costs about
+ * 0.1 x (tools + history), while a request without the tools reads nothing and pays the history in
+ * full: the tools pay their way while they are less than nine times the history.
+ */
+async function toolsRepayTheirPrefix(tools: plugins.ToolSet, messages: plugins.ModelMessage[]): Promise<boolean> {
+  let toolBytes = 0;
+  for (const [name, definition] of Object.entries(tools)) {
+    const schema = definition.type === 'provider' ? definition.args : await plugins.asSchema(definition.inputSchema).jsonSchema;
+    toolBytes += name.length + JSON.stringify([definition.description ?? '', schema ?? null]).length;
+  }
+  return toolBytes <= 9 * JSON.stringify(messages).length;
+}
+
+/** A tool set's definitions without anything that runs: no `execute`, no input callbacks. */
+function toolDefinitions(tools: plugins.ToolSet): plugins.ToolSet {
+  return Object.fromEntries(Object.entries(tools).map(([name, definition]) => {
+    const {
+      execute: _execute,
+      onInputStart: _onInputStart,
+      onInputDelta: _onInputDelta,
+      onInputAvailable: _onInputAvailable,
+      ...rest
+    } = definition as plugins.ToolSet[string] & Record<string, unknown>;
+    return [name, rest as plugins.ToolSet[string]];
+  }));
 }
 
 /**
  * Compacts a message history into a summary.
  *
  * The summary is one streamed, instructed model call, the shape every agent turn has, so it reaches
- * a provider that serves only that shape (the ChatGPT Codex backend) the way the turns do.
+ * a provider that serves only that shape (the ChatGPT Codex backend) the way the turns do. With the
+ * turns' `tools` it also repeats their prefix, which the provider has cached; a model that answers
+ * it with a tool call instead of a summary is asked once more with tool use turned off.
  *
  * @param model    The same model used by runAgent, or a cheaper small model
  * @param messages The full message history that overflowed
@@ -63,9 +104,12 @@ export async function compactMessages(
   messages: plugins.ModelMessage[],
   options: ICompactMessagesOptions = {},
 ): Promise<plugins.ModelMessage[]> {
+  const tools = options.tools && Object.keys(options.tools).length > 0 && await toolsRepayTheirPrefix(options.tools, messages)
+    ? toolDefinitions(options.tools)
+    : undefined;
   const prompt: plugins.ModelMessage[] = [
     ...messages,
-    { role: 'user', content: COMPACTION_PROMPT },
+    { role: 'user', content: tools ? `${COMPACTION_PROMPT}\n\n${NO_TOOL_CALLS}` : COMPACTION_PROMPT },
   ];
   const reportUsage = options.reportUsage;
   const modelCalls = reportUsage
@@ -89,38 +133,50 @@ export async function compactMessages(
         },
       })
     : model;
-  let streamError: unknown;
-  try {
-    const result = plugins.streamText({
-      model: observedModel,
-      instructions: options.system ?? COMPACTION_INSTRUCTIONS,
-      messages: prompt,
-      allowSystemInMessages: true,
-      ...(options.providerOptions ? { providerOptions: options.providerOptions } : {}),
-      abortSignal: options.abortSignal,
-      onError: ({ error }) => {
-        streamError ??= error;
-      },
-      ...(modelCalls
-        ? {
-            onLanguageModelCallEnd: ({ modelId, usage }) => {
-              modelCalls.end(modelId, usage);
-            },
-          }
-        : {}),
-    });
-    let summary = '';
-    for await (const delta of result.textStream) summary += delta;
-    // A stream that fails delivers its error to onError and ends, and an aborted stream just ends;
-    // either way the summary is incomplete and must not replace the history.
-    if (streamError !== undefined) throw streamError;
-    options.abortSignal?.throwIfAborted();
-    modelCalls?.settle('missing');
-    return summaryMessages(summary);
-  } catch (error) {
-    modelCalls?.settle(options.abortSignal?.aborted ? 'aborted' : 'failed');
-    throw streamError ?? error;
-  }
+  // The summary is asked for in the shape of a turn, the tools offered as the turns offer them. A
+  // model that calls a tool instead of answering is asked again with tool use turned off, which
+  // keeps the tool definitions and so still the cached tools and instructions.
+  const summarize = async (toolChoice?: 'none'): Promise<{ summary: string; calledTools: boolean }> => {
+    let streamError: unknown;
+    try {
+      const result = plugins.streamText({
+        model: observedModel,
+        instructions: options.system ?? COMPACTION_INSTRUCTIONS,
+        messages: prompt,
+        allowSystemInMessages: true,
+        ...(tools ? { tools, ...(toolChoice ? { toolChoice } : {}) } : {}),
+        ...(options.providerOptions ? { providerOptions: options.providerOptions } : {}),
+        abortSignal: options.abortSignal,
+        onError: ({ error }) => {
+          streamError ??= error;
+        },
+        ...(modelCalls
+          ? {
+              onLanguageModelCallEnd: ({ modelId, usage }) => {
+                modelCalls.end(modelId, usage);
+              },
+            }
+          : {}),
+      });
+      let summary = '';
+      for await (const delta of result.textStream) summary += delta;
+      // A stream that fails delivers its error to onError and ends, and an aborted stream just ends;
+      // either way the summary is incomplete and must not replace the history.
+      if (streamError !== undefined) throw streamError;
+      options.abortSignal?.throwIfAborted();
+      modelCalls?.settle('missing');
+      return { summary, calledTools: tools !== undefined && (await result.toolCalls).length > 0 };
+    } catch (error) {
+      modelCalls?.settle(options.abortSignal?.aborted ? 'aborted' : 'failed');
+      throw streamError ?? error;
+    }
+  };
+  // A provider-executed tool, such as a web search, would run on the provider's side before the
+  // retry could prevent it: with one in the set, tool use is off from the start.
+  const providerExecuted = tools !== undefined && Object.values(tools).some((definition) => definition.type === 'provider');
+  let { summary, calledTools } = await summarize(providerExecuted ? 'none' : undefined);
+  if (calledTools && !summary.trim()) ({ summary } = await summarize('none'));
+  return summaryMessages(summary);
 }
 
 function summaryMessages(summary: string): plugins.ModelMessage[] {