@modelprofile.com/flexharness 10.13.0 → 10.14.0

package/ts/plugins.ts

@@ -13,6 +13,84 @@ import { Validator as JsonSchemaValidator } from '@cfworker/json-schema';
 export { asSchema, jsonSchema, JsonSchemaValidator };
 export type { FlexibleSchema, JSONSchema7, ModelMessage, Schema } from 'ai';
 
+// optional code mode peers: QuickJS-ng compiled to WebAssembly, and amaro to strip TypeScript types
+
+/** A value inside a QuickJS context; the host disposes every handle it owns. */
+export interface IQuickJsHandle {
+  readonly alive: boolean;
+  dispose(): void;
+}
+
+export type TQuickJsResult =
+  | { value: IQuickJsHandle; error?: undefined }
+  | { error: IQuickJsHandle; value?: undefined };
+
+export interface IQuickJsDeferredPromise {
+  readonly handle: IQuickJsHandle;
+  readonly alive: boolean;
+  resolve(value?: IQuickJsHandle): void;
+  reject(value?: IQuickJsHandle): void;
+  dispose(): void;
+}
+
+export interface IQuickJsContext {
+  readonly global: IQuickJsHandle;
+  newString(value: string): IQuickJsHandle;
+  getString(handle: IQuickJsHandle): string;
+  newFunction(name: string, implementation: (...args: IQuickJsHandle[]) => IQuickJsHandle | void): IQuickJsHandle;
+  setProp(target: IQuickJsHandle, key: string, value: IQuickJsHandle): void;
+  newPromise(): IQuickJsDeferredPromise;
+  newError(message: string): IQuickJsHandle;
+  evalCode(code: string, filename?: string): TQuickJsResult;
+  resolvePromise(handle: IQuickJsHandle): Promise<TQuickJsResult>;
+  dump(handle: IQuickJsHandle): unknown;
+  dispose(): void;
+}
+
+export interface IQuickJsRuntime {
+  readonly alive: boolean;
+  setMemoryLimit(bytes: number): void;
+  setMaxStackSize(bytes: number): void;
+  setInterruptHandler(handler: () => boolean): void;
+  newContext(): IQuickJsContext;
+  /** Runs the queued promise jobs; `error` is an exception none of them caught. */
+  executePendingJobs(): { error?: IQuickJsHandle };
+  dispose(): void;
+}
+
+export interface IQuickJsModule {
+  newRuntime(): IQuickJsRuntime;
+}
+
+/** A failed strip, as amaro reports it. */
+export interface IAmaroError {
+  code?: string;
+  message?: string;
+  startLine?: number;
+}
+
+export interface ICodeSandboxEngine {
+  /** A new WebAssembly instance of QuickJS-ng, with a memory of its own. */
+  newModule(): Promise<IQuickJsModule>;
+  /** Removes TypeScript type syntax, keeping every line where it was. */
+  stripTypes(source: string): string;
+}
+
+/** Loads QuickJS and amaro, the optional peers code mode needs. */
+export const importCodeSandboxEngine = async (): Promise<ICodeSandboxEngine> => {
+  const [core, variant, amaro] = await Promise.all([
+    import('quickjs-emscripten-core'),
+    import('@jitl/quickjs-ng-wasmfile-release-sync'),
+    import('amaro'),
+  ]);
+  // The variant's typings resolve as CommonJS; at run time the default export is the variant.
+  const quickjsVariant = (variant as { default: unknown }).default as Parameters<typeof core.newQuickJSWASMModuleFromVariant>[0];
+  return {
+    newModule: async () => await core.newQuickJSWASMModuleFromVariant(quickjsVariant) as unknown as IQuickJsModule,
+    stripTypes: (source) => amaro.transformSync(source, { mode: 'strip-only' }).code,
+  };
+};
+
 // @push.rocks scope
 import {
   AgentEventStoreConflictError,
@@ -80,6 +158,7 @@ export type {
   TAgentGenerationOutcome,
   TAgentModelCallUsage,
   TAgentModelCallUsageEvent,
+  TAgentObservedModelCallUsage,
   TAgentRetryReason,
   TAgentToolCallFinishEvent,
   TAgentToolExecutionReconciliationOptions,

package/ts/readme.md

@@ -1026,7 +1026,7 @@ A `contextCompactor` states which model to compact with from the model the compa
 
 A compactor uses `resolvedModel.model` directly, and resolves `modelHint` the way its model resolver resolves a prompt's `modelHint`. Only when neither member is present, because nothing dispatched states a model, does the host choose the compaction model itself.
 
-A `context-overflow` or `context-budget` compaction also carries `system` and `providerOptions`: the instructions and the provider options (cache options applied, so `store: false` on OpenAI) the run's turns are sent with. `compactMessages()` sends its summary call with them as one streamed call, the shape of every turn, so a compactor that passes its options through reaches the provider the way the run does. Without `system` it sends its own instructions.
+A `context-overflow` or `context-budget` compaction also carries `system` and `providerOptions`: the instructions and the provider options (cache options applied, so `store: false` on OpenAI) the run's turns are sent with. A `context-budget` compaction on a provider that caches the prefix also carries the turns' `tools`. `compactMessages()` sends its summary call with them as one streamed call, the shape of every turn, with the tools' definitions but none of their code, so a compactor that passes its options through repeats the run's prefix and reads it from the provider's prompt cache; definitions that outweigh the history nine times are left out. Without `system` it sends its own instructions.
 
 ```typescript
 agentSessionPolicy: {

package/ts/utils.harnessoptions.ts

@@ -1,5 +1,7 @@
 import { FlexHarnessValidationError } from './errors.js';
 import {
+  FLEX_CODE_MODE_DEFAULT_LIMITS,
+  FLEX_CODE_MODE_MAXIMUM_LIMITS,
   FLEX_REVERSION_DEFAULT_LIMITS,
   FLEX_REVERSION_MAXIMUM_LIMITS,
 } from './interfaces.js';
@@ -12,6 +14,7 @@ import type {
   IFlexReversionLimits,
   IFlexSlashCommandHandlerRegistration,
   IFlexSlashCommandTemplateRegistration,
+  IFlexCodeModeLimits,
   IFlexSubagentDefinition,
   IFlexToolCatalogOptions,
   TFlexSlashCommandRegistration,
@@ -157,25 +160,55 @@ export function normalizeSubagents(
   return Object.freeze(normalized);
 }
 
+export interface INormalizedToolCatalogOptions {
+  readonly declarations: 'typescript' | 'json-schema';
+  /** The limits of `run_code`; absent while code mode is off. */
+  readonly codeMode?: Readonly<IFlexCodeModeLimits>;
+}
+
+const isPlainOptions = (value: unknown): value is Record<string, unknown> =>
+  !!value
+  && typeof value === 'object'
+  && !Array.isArray(value)
+  && [Object.prototype, null].includes(Object.getPrototypeOf(value));
+
 export function normalizeToolCatalogOptions(
-  options: IFlexHarnessOptions<unknown>['toolCatalog'],
-): Readonly<Required<IFlexToolCatalogOptions>> {
+  options: IFlexToolCatalogOptions | undefined,
+): INormalizedToolCatalogOptions {
   if (options === undefined) return Object.freeze({ declarations: 'typescript' });
-  if (
-    !options
-    || typeof options !== 'object'
-    || Array.isArray(options)
-    || ![Object.prototype, null].includes(Object.getPrototypeOf(options))
-  ) {
-    throw new FlexHarnessValidationError('toolCatalog must be a plain object.');
-  }
-  const unsupported = Object.keys(options).find((key) => key !== 'declarations');
+  if (!isPlainOptions(options)) throw new FlexHarnessValidationError('toolCatalog must be a plain object.');
+  const unsupported = Object.keys(options).find((key) => key !== 'declarations' && key !== 'codeMode');
   if (unsupported) throw new FlexHarnessValidationError(`toolCatalog does not support "${unsupported}".`);
   const declarations = options.declarations ?? 'typescript';
   if (declarations !== 'typescript' && declarations !== 'json-schema') {
     throw new FlexHarnessValidationError('toolCatalog.declarations must be "typescript" or "json-schema".');
   }
-  return Object.freeze({ declarations });
+  const codeMode = normalizeCodeModeLimits(options.codeMode);
+  return Object.freeze({ declarations, ...(codeMode ? { codeMode } : {}) });
+}
+
+function normalizeCodeModeLimits(codeMode: unknown): Readonly<IFlexCodeModeLimits> | undefined {
+  if (codeMode === undefined || codeMode === false) return undefined;
+  if (codeMode === true) return FLEX_CODE_MODE_DEFAULT_LIMITS;
+  if (!isPlainOptions(codeMode)) {
+    throw new FlexHarnessValidationError('toolCatalog.codeMode must be a boolean or a plain object.');
+  }
+  const unsupported = Object.keys(codeMode).find((key) => key !== 'limits');
+  if (unsupported) throw new FlexHarnessValidationError(`toolCatalog.codeMode does not support "${unsupported}".`);
+  const limits = codeMode.limits ?? {};
+  if (!isPlainOptions(limits)) throw new FlexHarnessValidationError('toolCatalog.codeMode.limits must be a plain object.');
+  const resolved: Record<string, number> = { ...FLEX_CODE_MODE_DEFAULT_LIMITS };
+  for (const [key, value] of Object.entries(limits)) {
+    if (!Object.prototype.hasOwnProperty.call(FLEX_CODE_MODE_MAXIMUM_LIMITS, key)) {
+      throw new FlexHarnessValidationError(`toolCatalog.codeMode.limits does not support "${key}".`);
+    }
+    const maximum = FLEX_CODE_MODE_MAXIMUM_LIMITS[key as keyof IFlexCodeModeLimits];
+    if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1 || value > maximum) {
+      throw new FlexHarnessValidationError(`toolCatalog.codeMode.limits.${key} must be an integer from 1 through ${maximum}.`);
+    }
+    resolved[key] = value;
+  }
+  return Object.freeze(resolved as unknown as IFlexCodeModeLimits);
 }
 
 export function normalizeBuiltInTools(

package/ts/utils.json.ts

@@ -785,6 +785,7 @@ function validateMessage(value: unknown, path: string): IValidatedMessageIdentit
           'output',
           'error',
           'childSessionId',
+          'parentToolCallId',
           'model',
           'startedAt',
           'completedAt',
@@ -801,6 +802,7 @@ function validateMessage(value: unknown, path: string): IValidatedMessageIdentit
       }
       requireOptionalString(part.error, `${partPath}.error`);
       requireOptionalString(part.childSessionId, `${partPath}.childSessionId`);
+      requireOptionalString(part.parentToolCallId, `${partPath}.parentToolCallId`);
       if (part.model !== undefined) validateModel(part.model, `${partPath}.model`);
       requireOptionalString(part.startedAt, `${partPath}.startedAt`);
       requireOptionalString(part.completedAt, `${partPath}.completedAt`);

package/ts/utils.tooloutput.ts

@@ -25,6 +25,15 @@ async function* normalizeAsyncIterable(
   }
 }
 
+/** A tool output's final value: the value itself, or the last value an async iterable yields. */
+export async function settleToolOutput(output: unknown): Promise<unknown> {
+  const value = await output;
+  if (!isAsyncIterable(value)) return value;
+  let last: unknown;
+  for await (const item of value) last = item;
+  return last;
+}
+
 export function wrapToolSet(
   tools: TFlexAgentToolSet,
   limits: Required<NonNullable<IFlexHarnessOptions<unknown>['toolOutputLimits']>>,

package/ts_compaction/plugins.ts

@@ -1,8 +1,8 @@
-import { streamText, wrapLanguageModel } from 'ai';
+import { asSchema, streamText, wrapLanguageModel } from 'ai';
 
-export { streamText, wrapLanguageModel };
+export { asSchema, streamText, wrapLanguageModel };
 
-export type { ModelMessage } from 'ai';
+export type { ModelMessage, ToolSet } from 'ai';
 export type { TSmartAiLanguageModel } from '@modelprofile.com/flexharness-models';
 
 import { AgentModelCallUsageRecorder } from '@modelprofile.com/flexharness-agent';

package/ts_compaction/smartagent.compaction.ts

@@ -25,6 +25,9 @@ When constructing the summary, try to stick to this template:
 /** The instructions of a summary call whose caller names none of the conversation's own. */
 const COMPACTION_INSTRUCTIONS = 'You summarize a conversation so that another agent can continue its work.';
 
+/** Added to the summary request when it carries the conversation's tools. */
+const NO_TOOL_CALLS = 'Do not call any tool. Reply with the summary text only.';
+
 export interface ICompactMessagesOptions {
   abortSignal?: AbortSignal;
   /**
@@ -45,13 +48,51 @@ export interface ICompactMessagesOptions {
    * options' `providerOptions` through.
    */
   providerOptions?: plugins.ProviderOptions;
+  /**
+   * The tools the conversation's turns are sent with. The summary call carries their definitions,
+   * none of their code, so its request repeats the turns' prefix and reads it from the provider's
+   * prompt cache, unless the definitions outweigh the history nine times, when reading them would
+   * cost more than the history saves. A context compactor passes its options' `tools` through.
+   */
+  tools?: plugins.ToolSet;
+}
+
+/**
+ * Whether the summary request should carry the turns' tool definitions. A cached prefix is read at
+ * about a tenth of the input price (Anthropic and OpenAI), so repeating the turns' prefix costs about
+ * 0.1 x (tools + history), while a request without the tools reads nothing and pays the history in
+ * full: the tools pay their way while they are less than nine times the history.
+ */
+async function toolsRepayTheirPrefix(tools: plugins.ToolSet, messages: plugins.ModelMessage[]): Promise<boolean> {
+  let toolBytes = 0;
+  for (const [name, definition] of Object.entries(tools)) {
+    const schema = definition.type === 'provider' ? definition.args : await plugins.asSchema(definition.inputSchema).jsonSchema;
+    toolBytes += name.length + JSON.stringify([definition.description ?? '', schema ?? null]).length;
+  }
+  return toolBytes <= 9 * JSON.stringify(messages).length;
+}
+
+/** A tool set's definitions without anything that runs: no `execute`, no input callbacks. */
+function toolDefinitions(tools: plugins.ToolSet): plugins.ToolSet {
+  return Object.fromEntries(Object.entries(tools).map(([name, definition]) => {
+    const {
+      execute: _execute,
+      onInputStart: _onInputStart,
+      onInputDelta: _onInputDelta,
+      onInputAvailable: _onInputAvailable,
+      ...rest
+    } = definition as plugins.ToolSet[string] & Record<string, unknown>;
+    return [name, rest as plugins.ToolSet[string]];
+  }));
 }
 
 /**
  * Compacts a message history into a summary.
  *
  * The summary is one streamed, instructed model call, the shape every agent turn has, so it reaches
- * a provider that serves only that shape (the ChatGPT Codex backend) the way the turns do.
+ * a provider that serves only that shape (the ChatGPT Codex backend) the way the turns do. With the
+ * turns' `tools` it also repeats their prefix, which the provider has cached; a model that answers
+ * it with a tool call instead of a summary is asked once more with tool use turned off.
  *
  * @param model    The same model used by runAgent, or a cheaper small model
  * @param messages The full message history that overflowed
@@ -63,9 +104,12 @@ export async function compactMessages(
   messages: plugins.ModelMessage[],
   options: ICompactMessagesOptions = {},
 ): Promise<plugins.ModelMessage[]> {
+  const tools = options.tools && Object.keys(options.tools).length > 0 && await toolsRepayTheirPrefix(options.tools, messages)
+    ? toolDefinitions(options.tools)
+    : undefined;
   const prompt: plugins.ModelMessage[] = [
     ...messages,
-    { role: 'user', content: COMPACTION_PROMPT },
+    { role: 'user', content: tools ? `${COMPACTION_PROMPT}\n\n${NO_TOOL_CALLS}` : COMPACTION_PROMPT },
   ];
   const reportUsage = options.reportUsage;
   const modelCalls = reportUsage
@@ -89,38 +133,50 @@ export async function compactMessages(
         },
       })
     : model;
-  let streamError: unknown;
-  try {
-    const result = plugins.streamText({
-      model: observedModel,
-      instructions: options.system ?? COMPACTION_INSTRUCTIONS,
-      messages: prompt,
-      allowSystemInMessages: true,
-      ...(options.providerOptions ? { providerOptions: options.providerOptions } : {}),
-      abortSignal: options.abortSignal,
-      onError: ({ error }) => {
-        streamError ??= error;
-      },
-      ...(modelCalls
-        ? {
-            onLanguageModelCallEnd: ({ modelId, usage }) => {
-              modelCalls.end(modelId, usage);
-            },
-          }
-        : {}),
-    });
-    let summary = '';
-    for await (const delta of result.textStream) summary += delta;
-    // A stream that fails delivers its error to onError and ends, and an aborted stream just ends;
-    // either way the summary is incomplete and must not replace the history.
-    if (streamError !== undefined) throw streamError;
-    options.abortSignal?.throwIfAborted();
-    modelCalls?.settle('missing');
-    return summaryMessages(summary);
-  } catch (error) {
-    modelCalls?.settle(options.abortSignal?.aborted ? 'aborted' : 'failed');
-    throw streamError ?? error;
-  }
+  // The summary is asked for in the shape of a turn, the tools offered as the turns offer them. A
+  // model that calls a tool instead of answering is asked again with tool use turned off, which
+  // keeps the tool definitions and so still the cached tools and instructions.
+  const summarize = async (toolChoice?: 'none'): Promise<{ summary: string; calledTools: boolean }> => {
+    let streamError: unknown;
+    try {
+      const result = plugins.streamText({
+        model: observedModel,
+        instructions: options.system ?? COMPACTION_INSTRUCTIONS,
+        messages: prompt,
+        allowSystemInMessages: true,
+        ...(tools ? { tools, ...(toolChoice ? { toolChoice } : {}) } : {}),
+        ...(options.providerOptions ? { providerOptions: options.providerOptions } : {}),
+        abortSignal: options.abortSignal,
+        onError: ({ error }) => {
+          streamError ??= error;
+        },
+        ...(modelCalls
+          ? {
+              onLanguageModelCallEnd: ({ modelId, usage }) => {
+                modelCalls.end(modelId, usage);
+              },
+            }
+          : {}),
+      });
+      let summary = '';
+      for await (const delta of result.textStream) summary += delta;
+      // A stream that fails delivers its error to onError and ends, and an aborted stream just ends;
+      // either way the summary is incomplete and must not replace the history.
+      if (streamError !== undefined) throw streamError;
+      options.abortSignal?.throwIfAborted();
+      modelCalls?.settle('missing');
+      return { summary, calledTools: tools !== undefined && (await result.toolCalls).length > 0 };
+    } catch (error) {
+      modelCalls?.settle(options.abortSignal?.aborted ? 'aborted' : 'failed');
+      throw streamError ?? error;
+    }
+  };
+  // A provider-executed tool, such as a web search, would run on the provider's side before the
+  // retry could prevent it: with one in the set, tool use is off from the start.
+  const providerExecuted = tools !== undefined && Object.values(tools).some((definition) => definition.type === 'provider');
+  let { summary, calledTools } = await summarize(providerExecuted ? 'none' : undefined);
+  if (calledTools && !summary.trim()) ({ summary } = await summarize('none'));
+  return summaryMessages(summary);
 }
 
 function summaryMessages(summary: string): plugins.ModelMessage[] {