@modelprofile.com/flexharness 10.13.0 → 10.14.0

package/ts/classes.flexharness.ts

@@ -28,6 +28,7 @@ import {
   FLEX_REVERSION_REFERENCE_MAX_BYTES,
   FLEX_REVERSION_WORKSPACE_ID_MAX_BYTES,
   FLEX_REVERSION_WORKSPACE_LABEL_MAX_BYTES,
+  FLEX_CODE_MODE_TOOL_NAME,
   FLEX_SESSION_IMPORT_MAX_DEPTH,
   FLEX_TOOL_CATALOG_TOOL_NAMES,
 } from './interfaces.js';
@@ -36,6 +37,7 @@ import {
   assertNoCatalogCollisions,
   catalogToolInputSchema,
 } from './classes.toolcatalog.js';
+import { FlexCodeSandbox, type IFlexCodeHost } from './classes.codesandbox.js';
 import type {
   IFlexAgentContextCompactionOptions,
   IFlexAgentContextInvocation,
@@ -119,6 +121,7 @@ import type {
   IFlexSubagentProvenance,
   IFlexTerminalProjection,
   IFlexCatalogDescribeInput,
+  IFlexCatalogProposal,
   IFlexCatalogSearchInput,
   IFlexToolCatalogOptions,
   IFlexToolHandle,
@@ -222,7 +225,7 @@ import {
   validateSessionImportHandle,
   type INormalizedSessionImportOptions,
 } from './utils.sessionimport.js';
-import { wrapToolSet } from './utils.tooloutput.js';
+import { settleToolOutput, wrapToolSet } from './utils.tooloutput.js';
 import {
   validateCompactSessionOptions,
   validateCreateProjectTaskInput,
@@ -445,6 +448,8 @@ interface IActiveRun {
   toolProviderContext?: Readonly<IFlexToolProviderContext<unknown>>;
   /** The catalog methods the run offers through search_tools, describe_tools and execute. */
   toolCatalog?: FlexToolCatalog<unknown>;
+  /** The sandbox `run_code` programs run in, loaded when the catalog has code mode. */
+  codeSandbox?: FlexCodeSandbox;
   subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
   allowedTools?: readonly string[];
   agentAllowedTools?: readonly string[];
@@ -865,7 +870,7 @@ export class FlexHarness<TScope = unknown> {
   private readonly externalErrorProjector?: TFlexExternalErrorProjector;
   private readonly subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
   private readonly builtInTools: INormalizedBuiltInTools;
-  private readonly toolCatalogOptions: Readonly<Required<IFlexToolCatalogOptions>>;
+  private readonly toolCatalogOptions: Readonly<IFlexToolCatalogOptions>;
   private readonly slashCommands: ReadonlyMap<string, TRegisteredSlashCommand<TScope>>;
   private readonly maxSubagentDepth: number;
   private readonly maxSubagentCallsPerRun: number;
@@ -952,7 +957,11 @@ export class FlexHarness<TScope = unknown> {
     this.externalErrorProjector = options.externalErrorProjector;
     this.subagents = normalizeSubagents(options.subagents);
     this.builtInTools = normalizeBuiltInTools(options.builtInTools);
-    this.toolCatalogOptions = normalizeToolCatalogOptions(options.toolCatalog);
+    const toolCatalogOptions = normalizeToolCatalogOptions(options.toolCatalog);
+    this.toolCatalogOptions = Object.freeze({
+      declarations: toolCatalogOptions.declarations,
+      ...(toolCatalogOptions.codeMode ? { codeMode: { limits: toolCatalogOptions.codeMode } } : {}),
+    });
     this.slashCommands = normalizeSlashCommands(options.slashCommands);
     this.maxSubagentDepth = resolveBoundedPositiveInteger(
       options.maxSubagentDepth,
@@ -5844,6 +5853,7 @@ export class FlexHarness<TScope = unknown> {
       if (toolHandle?.catalog) {
         try {
           catalog = await FlexToolCatalog.create(toolHandle.catalog, this.toolCatalogOptions);
+          if (catalog.codeMode) run.codeSandbox = await FlexCodeSandbox.load();
         } catch (error) {
           throw this.projectExternalError(run, error, 'toolProvider');
         }
@@ -5984,20 +5994,141 @@ export class FlexHarness<TScope = unknown> {
           input: { method: string; params?: Record<string, unknown> },
           options?: { toolCallId?: string; abortSignal?: AbortSignal },
         ) => {
+          const toolCallId = options?.toolCallId ?? '';
+          const abortSignal = options?.abortSignal ?? run.controller.signal;
           const call = await catalog.prepareCall(input.method, input.params);
-          if (!call.success) {
-            const error = this.trustInternalError(new FlexHarnessValidationError(JSON.stringify(call.error)));
-            if (options?.toolCallId) run.trustedToolExecutionErrors.set(options.toolCallId, error);
-            throw error;
+          if (!call.success) throw this.catalogCallError(run, toolCallId, call.error);
+          const context = { method: call.method, toolCallId, abortSignal, provider: run.toolProviderContext! };
+          const decision = await call.decide(context);
+          if (decision === 'run') return call.run(context);
+          if (decision === 'propose') {
+            return catalog.propose(
+              [{ id: toolCallId, method: call.method, params: call.params }],
+              { toolCallId, abortSignal, provider: run.toolProviderContext! },
+            );
           }
-          return call.run({
-            method: call.method,
-            toolCallId: options?.toolCallId ?? '',
-            abortSignal: options?.abortSignal ?? run.controller.signal,
-            provider: run.toolProviderContext!,
-          });
+          throw this.catalogCallError(run, toolCallId, { error: 'denied', method: call.method, reason: decision.deny });
         },
       }),
+      ...(catalog.codeMode && run.codeSandbox ? {
+        [FLEX_CODE_MODE_TOOL_NAME]: plugins.tool({
+          description: definitions[FLEX_CODE_MODE_TOOL_NAME].description,
+          strict: false,
+          inputSchema: catalogToolInputSchema<{ code: string }>(definitions[FLEX_CODE_MODE_TOOL_NAME]),
+          execute: async (input: { code: string }, options?: { toolCallId?: string; abortSignal?: AbortSignal }) =>
+            this.runCatalogProgram(run, catalog, input.code, options?.toolCallId ?? '', options?.abortSignal ?? run.controller.signal),
+        }),
+      } : {}),
+    };
+  }
+
+  /** A refused catalog call as the trusted error the model and the tool part show. */
+  private catalogCallError(run: IActiveRun, toolCallId: string, payload: unknown): Error {
+    const error = this.trustInternalError(new FlexHarnessValidationError(JSON.stringify(payload)));
+    if (toolCallId) run.trustedToolExecutionErrors.set(toolCallId, error);
+    return error;
+  }
+
+  /**
+   * Runs a `run_code` program. Every catalog call it makes is dispatched like an `execute`
+   * (validation, `onCall`, the method's run with its permission requests, the output limits, error
+   * projection) and recorded as a tool part of its own, named by the method, with the `run_code`
+   * call as its parent. Proposals go to `onProposals` as one batch once the program finished; a
+   * program that fails proposes nothing.
+   */
+  private async runCatalogProgram(
+    run: IActiveRun,
+    catalog: FlexToolCatalog<unknown>,
+    code: string,
+    toolCallId: string,
+    abortSignal: AbortSignal,
+  ): Promise<unknown> {
+    const proposals: IFlexCatalogProposal[] = [];
+    const base = run.toolProviderContext!;
+    // Waiting for a permission decision is the user's time, not the program's.
+    let waitedMs = 0;
+    let waiting = 0;
+    let waitingSince = 0;
+    /** The provider context of one call: its permission waits are timed, and end when the program stops. */
+    const providerFor = (signal: AbortSignal): Readonly<IFlexToolProviderContext<unknown>> => Object.freeze({
+      ...base,
+      requestPermission: async (request: IFlexPermissionRequestInput) => {
+        signal.throwIfAborted();
+        if (waiting++ === 0) waitingSince = performance.now();
+        let onAbort: (() => void) | undefined;
+        try {
+          await Promise.race([
+            base.requestPermission(request),
+            new Promise<never>((_resolve, reject) => {
+              onAbort = () => reject(signal.reason);
+              signal.addEventListener('abort', onAbort, { once: true });
+            }),
+          ]);
+        } finally {
+          if (onAbort) signal.removeEventListener('abort', onAbort);
+          if (--waiting === 0) waitedMs += performance.now() - waitingSince;
+        }
+      },
+    });
+    const stopped = (method: string) => ({ ok: false as const, error: { error: 'aborted', method } });
+    let callNumber = 0;
+    const finish = (id: string, method: string, input: unknown, outcome: { output: unknown } | { error: Error }) => {
+      if ('error' in outcome) run.trustedToolExecutionErrors.set(id, outcome.error);
+      this.onToolFinish(run.state, run.sessionId, 'error' in outcome
+        ? { toolCallId: id, toolName: method, input, success: false, error: outcome.error.message }
+        : { toolCallId: id, toolName: method, input, success: true, output: outcome.output });
+    };
+    const host: IFlexCodeHost = {
+      waitedMs: () => waitedMs + (waiting > 0 ? performance.now() - waitingSince : 0),
+      call: async (method, params, signal) => {
+        if (signal.aborted) return stopped(method);
+        const id = `${toolCallId}.${++callNumber}`;
+        const call = await catalog.prepareCall(method, params);
+        if (!call.success) {
+          if (catalog.has(method)) {
+            this.onToolStart(run.state, run.sessionId, { toolCallId: id, toolName: method, input: params ?? {}, parentToolCallId: toolCallId });
+            finish(id, method, params, { error: this.catalogCallError(run, id, call.error) });
+          }
+          return { ok: false, error: call.error as unknown as TJsonValue };
+        }
+        if (signal.aborted) return stopped(call.method);
+        this.onToolStart(run.state, run.sessionId, {
+          toolCallId: id, toolName: call.method, input: call.params, parentToolCallId: toolCallId,
+        });
+        const context = { method: call.method, toolCallId: id, abortSignal: signal, provider: providerFor(signal) };
+        try {
+          const decision = await call.decide(context);
+          signal.throwIfAborted();
+          if (decision === 'propose') {
+            proposals.push({ id, method: call.method, params: call.params });
+            const value = { proposed: true, id };
+            finish(id, call.method, call.params, { output: value });
+            return { ok: true, value };
+          }
+          if (decision !== 'run') {
+            const payload = { error: 'denied' as const, method: call.method, reason: decision.deny };
+            finish(id, call.method, call.params, { error: this.catalogCallError(run, id, payload) });
+            return { ok: false, error: payload };
+          }
+          const value = normalizeJsonValue(await settleToolOutput(call.run(context)), this.toolOutputLimits);
+          finish(id, call.method, call.params, { output: value });
+          return { ok: true, value };
+        } catch (error) {
+          const projected = this.projectExternalError(run, error, 'toolExecution');
+          finish(id, call.method, call.params, { error: projected });
+          return { ok: false, error: { error: 'failed', method: call.method, message: projected.message } };
+        }
+      },
+    };
+    const result = await run.codeSandbox!.run(code, host, catalog.codeMode!, abortSignal);
+    if (!result.ok) throw this.catalogCallError(run, toolCallId, result.failure);
+    return {
+      result: result.value,
+      calls: result.calls,
+      ...(result.logs ? { logs: result.logs } : {}),
+      ...(proposals.length > 0
+        ? { proposals: await catalog.propose(proposals, { toolCallId, abortSignal, provider: base }) }
+        : {}),
     };
   }
 
@@ -8505,7 +8636,7 @@ export class FlexHarness<TScope = unknown> {
   private onToolStart(
     state: IStorageState,
     sessionId: string,
-    event: { toolCallId: string; toolName: string; input: unknown },
+    event: { toolCallId: string; toolName: string; input: unknown; parentToolCallId?: string },
   ): void {
     const run = this.callbackRun(state, sessionId);
     if (!run || run.toolPartIds.has(event.toolCallId)) return;
@@ -8523,6 +8654,9 @@ export class FlexHarness<TScope = unknown> {
         toolName: this.boundedIdentifier(target ? target.method : event.toolName, 'toolName'),
         status: 'running',
         input,
+        ...(event.parentToolCallId === undefined
+          ? {}
+          : { parentToolCallId: this.boundedIdentifier(event.parentToolCallId, 'parentToolCallId') }),
         startedAt: new Date().toISOString(),
       };
       run.toolPartIds.set(event.toolCallId, part.partId);

package/ts/classes.toolcatalog.ts

@@ -1,8 +1,13 @@
 import * as plugins from './plugins.js';
 import { FlexHarnessValidationError } from './errors.js';
 import {
+  FLEX_CODE_MODE_TOOL_NAME,
   FLEX_TOOL_CATALOG_LIMITS as limits,
   FLEX_TOOL_CATALOG_TOOL_NAMES as toolNames,
+  type IFlexCatalogProposal,
+  type IFlexCatalogProposalContext,
+  type IFlexCodeModeLimits,
+  type TFlexCatalogCallDecision,
   type IFlexCatalogDescribeInput,
   type IFlexCatalogDescribeResult,
   type IFlexCatalogMethod,
@@ -25,6 +30,7 @@ import {
   type IRenderedDeclaration,
   type TJsonSchemaNode,
 } from './utils.schemadeclarations.js';
+import { normalizeToolCatalogOptions } from './utils.harnessoptions.js';
 import { validateUtf8String } from './utils.validation.js';
 
 type TValidationResult = { success: true; value: unknown } | { success: false; error: Error };
@@ -58,6 +64,7 @@ interface IMethodEntry {
   readonly outputScope?: ISchemaScope;
   readonly declaration?: string;
   readonly run: IFlexCatalogMethod['run'];
+  readonly onCall?: NonNullable<IFlexCatalogMethod['onCall']>;
 }
 
 interface ITypeEntry {
@@ -76,6 +83,8 @@ interface IPreparedCall {
   readonly success: true;
   readonly method: string;
   readonly params: unknown;
+  /** The method's `onCall` decision for the call; `run` without the hook. */
+  decide(context: IFlexCatalogRunContext<any>): Promise<TFlexCatalogCallDecision>;
   run(context: IFlexCatalogRunContext<any>): ReturnType<IFlexCatalogMethod['run']>;
 }
 
@@ -99,10 +108,10 @@ const controlPattern = /[\u0000-\u001f\u007f]/;
 const definitionRefPattern = /^#\/(\$defs|definitions)\/([^/]+)$/;
 const declaredTypesHeader = '// Declares types: ';
 const identifierScanPattern = /[A-Za-z_$][A-Za-z0-9_$]*/g;
-const describeFormats = new Set(['typescript', 'json-schema']);
 const methodKeys = new Set([
-  'method', 'summary', 'description', 'tags', 'examples', 'inputSchema', 'outputSchema', 'declaration', 'run',
+  'method', 'summary', 'description', 'tags', 'examples', 'inputSchema', 'outputSchema', 'declaration', 'run', 'onCall',
 ]);
+const catalogKeys = new Set(['methods', 'types', 'onProposals']);
 const typeKeys = new Set(['name', 'description', 'schema', 'declaration']);
 const stopWords = new Set([
   'a', 'an', 'and', 'are', 'all', 'as', 'be', 'by', 'for', 'from', 'get', 'i', 'in', 'is', 'it', 'me', 'my',
@@ -421,14 +430,19 @@ class FlexToolCatalogCore {
     public readonly mode: TDeclarationMode,
     public readonly methods: ReadonlyMap<string, IMethodEntry>,
     public readonly types: ReadonlyMap<string, ITypeEntry>,
+    public readonly onProposals: IFlexToolCatalog['onProposals'] | undefined,
   ) {
     this.index = new CatalogSearchIndex([...methods.values()]);
   }
 
   public static async build(catalog: unknown, mode: TDeclarationMode): Promise<FlexToolCatalogCore> {
     if (!isPlainObject(catalog)) throw new FlexHarnessValidationError('A tool catalog must be a plain object.');
-    const unsupported = Object.keys(catalog).find((key) => key !== 'methods' && key !== 'types');
+    const unsupported = Object.keys(catalog).find((key) => !catalogKeys.has(key));
     if (unsupported) throw new FlexHarnessValidationError(`A tool catalog does not support "${unsupported}".`);
+    const onProposals = catalog.onProposals;
+    if (onProposals !== undefined && typeof onProposals !== 'function') {
+      throw new FlexHarnessValidationError('A tool catalog onProposals must be a function.');
+    }
     if (!Array.isArray(catalog.methods) || catalog.methods.length > limits.maxMethods) {
       throw new FlexHarnessValidationError(`Tool catalog methods must be an array with at most ${limits.maxMethods} entries.`);
     }
@@ -506,7 +520,14 @@ class FlexToolCatalogCore {
       });
     }
     const types = new Map([...registry.entries()].sort(([left], [right]) => compareCodePoints(left, right)));
-    return new FlexToolCatalogCore(mode, methods, types);
+    return new FlexToolCatalogCore(
+      mode,
+      methods,
+      types,
+      onProposals === undefined
+        ? undefined
+        : (proposals, context) => (onProposals as NonNullable<IFlexToolCatalog['onProposals']>).call(catalog, proposals, context),
+    );
   }
 
   private static async normalizeType(value: unknown, label: string): Promise<ITypeEntry> {
@@ -590,6 +611,9 @@ class FlexToolCatalogCore {
       throw new FlexHarnessValidationError(`${methodLabel} declaration must declare "function ${name}(".`);
     }
     if (typeof value.run !== 'function') throw new FlexHarnessValidationError(`${methodLabel} run must be a function.`);
+    if (value.onCall !== undefined && typeof value.onCall !== 'function') {
+      throw new FlexHarnessValidationError(`${methodLabel} onCall must be a function.`);
+    }
     if (value.inputSchema === undefined) throw new FlexHarnessValidationError(`${methodLabel} needs an inputSchema.`);
     const input = await resolveCatalogSchema(value.inputSchema, `${methodLabel} inputSchema`);
     const inputJson = input.json;
@@ -604,6 +628,7 @@ class FlexToolCatalogCore {
       ? undefined
       : await resolveCatalogSchema(value.outputSchema, `${methodLabel} outputSchema`);
     const run = value.run as IFlexCatalogMethod['run'];
+    const onCall = value.onCall as IFlexCatalogMethod['onCall'];
     return {
       path,
       namespace: segments.slice(0, -1).join('.'),
@@ -616,6 +641,9 @@ class FlexToolCatalogCore {
       ...(output ? { output } : {}),
       ...(declaration === undefined ? {} : { declaration }),
       run: (params, context) => run.call(value, params, context),
+      ...(onCall === undefined ? {} : {
+        onCall: (params: unknown, context: IFlexCatalogRunContext) => onCall.call(value, params, context),
+      }),
     };
   }
 
@@ -791,6 +819,8 @@ export class FlexToolCatalog<TScope = unknown> {
   private constructor(
     private readonly core: FlexToolCatalogCore,
     private readonly admitted: ReadonlySet<string> | undefined,
+    /** The limits of `run_code`; absent while code mode is off. */
+    public readonly codeMode: Readonly<IFlexCodeModeLimits> | undefined,
   ) {}
 
   /**
@@ -801,10 +831,8 @@ export class FlexToolCatalog<TScope = unknown> {
     catalog: IFlexToolCatalog<TScope>,
     options: IFlexToolCatalogOptions = {},
   ): Promise<FlexToolCatalog<TScope>> {
-    const mode = options.declarations ?? 'typescript';
-    if (!describeFormats.has(mode)) {
-      throw new FlexHarnessValidationError('toolCatalog.declarations must be "typescript" or "json-schema".');
-    }
+    const normalized = normalizeToolCatalogOptions(options);
+    const mode = normalized.declarations;
     if (typeof catalog !== 'object' || catalog === null) {
       throw new FlexHarnessValidationError('A tool catalog must be a plain object.');
     }
@@ -818,7 +846,7 @@ export class FlexToolCatalog<TScope = unknown> {
       core = FlexToolCatalogCore.build(catalog, mode);
       byMode.set(mode, core);
     }
-    return new FlexToolCatalog<TScope>(await core, undefined);
+    return new FlexToolCatalog<TScope>(await core, undefined, normalized.codeMode);
   }
 
   /** Every method path this catalog offers, sorted. */
@@ -837,6 +865,7 @@ export class FlexToolCatalog<TScope = unknown> {
     return new FlexToolCatalog<TScope>(
       this.core,
       new Set([...this.core.methods.keys()].filter((path) => allowed.has(path) && this.isAdmitted(path))),
+      this.codeMode,
     );
   }
 
@@ -844,7 +873,10 @@ export class FlexToolCatalog<TScope = unknown> {
     return this.core.methods.has(path) && this.isAdmitted(path);
   }
 
-  /** The three built-in tools as the model sees them. Their text depends only on the catalog. */
+  /**
+   * The built-in tools as the model sees them: the three catalog tools, and `run_code` in code
+   * mode. Their text depends only on the catalog and its options.
+   */
   public toolDefinitions(): Record<string, IFlexCatalogToolDefinition> {
     const typescript = this.core.mode === 'typescript';
     const methodPath = { type: 'string', maxLength: limits.maxMethodPathCharacters } as const;
@@ -899,9 +931,42 @@ export class FlexToolCatalog<TScope = unknown> {
           additionalProperties: false,
         },
       },
+      ...(this.codeMode ? { [FLEX_CODE_MODE_TOOL_NAME]: this.codeToolDefinition(this.codeMode) } : {}),
     };
   }
 
+  private codeToolDefinition(codeMode: Readonly<IFlexCodeModeLimits>): IFlexCatalogToolDefinition {
+    return {
+      description: [
+        'Run a short TypeScript or JavaScript program against the catalog, for work that takes several calls:',
+        'loops, filters, joins. Call a method as `await api.<dotted path>(params)`, for example',
+        '`await api.banking.listMovements({ accountId })`, with the declarations describe_tools returns; a refused',
+        'call throws an `ApiError` whose `details` say why. `return` a JSON value: it and the `console.log` output',
+        'come back, nothing else does. No network, files, timers or imports;',
+        `at most ${codeMode.maxCalls} calls and ${Math.round(codeMode.wallMs / 1000)} seconds.`,
+      ].join(' '),
+      inputSchema: {
+        type: 'object',
+        properties: { code: { type: 'string', minLength: 1, maxLength: codeMode.maxSourceBytes } },
+        required: ['code'],
+        additionalProperties: false,
+      },
+    };
+  }
+
+  /** Hands proposed calls to the catalog's `onProposals` and returns what the model sees for them. */
+  public async propose(
+    proposals: readonly IFlexCatalogProposal[],
+    context: IFlexCatalogProposalContext<TScope>,
+  ): Promise<unknown> {
+    if (!this.core.onProposals) {
+      throw new FlexHarnessValidationError(
+        `Tool catalog method "${proposals[0]?.method}" proposed a call, but the catalog has no onProposals.`,
+      );
+    }
+    return await this.core.onProposals(proposals, context as IFlexCatalogProposalContext);
+  }
+
   /** Search results or a namespace listing as text. */
   public search(input: IFlexCatalogSearchInput = {}): string {
     const limit = input.limit ?? limits.defaultSearchResults;
@@ -1041,6 +1106,19 @@ export class FlexToolCatalog<TScope = unknown> {
       success: true,
       method: path,
       params: validated,
+      decide: async (context) => {
+        const decision = entry.onCall ? await entry.onCall(validated, context) : 'run';
+        if (decision === 'run' || decision === 'propose') return decision;
+        if (
+          isSchemaRecord(decision)
+          && Object.keys(decision).length === 1
+          && typeof decision.deny === 'string'
+          && decision.deny.trim()
+        ) {
+          return { deny: decision.deny.slice(0, 2000) };
+        }
+        throw new FlexHarnessValidationError(`Tool catalog method "${path}" onCall returned an invalid decision.`);
+      },
       run: (context) => entry.run(validated, context),
     };
   }
@@ -1383,14 +1461,17 @@ const constraintText = (subset: Record<string, unknown>): string => {
 };
 
 /**
- * While a run has a catalog, its three tool names are reserved, and a method path must differ from
- * every tool name.
+ * While a run has a catalog, its tool names are reserved (`run_code` in code mode too), and a
+ * method path must differ from every tool name.
  */
 export const assertNoCatalogCollisions = (
   catalog: FlexToolCatalog<unknown>,
   tools: Readonly<Record<string, unknown>>,
 ): void => {
-  const catalogToolNames = new Set<string>(Object.values(toolNames));
+  const catalogToolNames = new Set<string>([
+    ...Object.values(toolNames),
+    ...(catalog.codeMode ? [FLEX_CODE_MODE_TOOL_NAME] : []),
+  ]);
   for (const name of catalogToolNames) {
     if (Object.prototype.hasOwnProperty.call(tools, name)) {
       throw new FlexHarnessValidationError(`Tool provider cannot define enabled built-in tool "${name}".`);

package/ts/interfaces.ts

@@ -31,6 +31,8 @@ export type TFlexRetryReason = plugins.TAgentRetryReason;
  * Every call carries `provider` and `requestedModelId`; key usage caps on those two.
  */
 export type TFlexModelCallUsage = plugins.TAgentModelCallUsage;
+/** A model call's usage as harness events carry it: a reported call also states its cache hit ratio. */
+export type TFlexObservedModelCallUsage = plugins.TAgentObservedModelCallUsage;
 
 export interface IFlexTextPromptPart {
   type: 'text';
@@ -135,6 +137,8 @@ export interface IFlexToolMessagePart {
   output?: TJsonValue;
   error?: string;
   childSessionId?: string;
+  /** For a catalog call a `run_code` program made: the `run_code` call's ID. */
+  parentToolCallId?: string;
   model?: IFlexModelIdentity;
   /** ISO time the tool call began. Absent on parts recorded before part times existed. */
   startedAt?: string;
@@ -743,6 +747,11 @@ export type TFlexCatalogCallError =
       more?: number;
       /** The lookup that returns the method's declaration. */
       see: string;
+    }
+  | {
+      error: 'denied';
+      method: string;
+      reason: string;
     };
 
 export interface IFlexCatalogToolDefinition {
@@ -761,7 +770,10 @@ export type TFlexCatalogSchema = { readonly [keyword: string]: unknown } | plugi
 export interface IFlexCatalogRunContext<TScope = unknown> {
   /** The dotted path the model called. */
   readonly method: string;
-  /** The `execute` call's ID; pass it with permission requests. */
+  /**
+   * The call's ID, which its tool part and permission requests carry: the `execute` call's, or for
+   * a call inside `run_code` the inner call's, `<run_code call ID>.<n>`.
+   */
   readonly toolCallId: string;
   readonly abortSignal: AbortSignal;
   /** The run's tool-provider context: scope, session, run and `requestPermission`. */
@@ -797,6 +809,31 @@ export interface IFlexCatalogMethod<TScope = unknown, TParams = any, TResult = u
     params: TParams,
     context: IFlexCatalogRunContext<TScope>,
   ): TResult | PromiseLike<TResult> | AsyncIterable<TResult>;
+  /**
+   * Decides each validated call before it runs: `run` (the default without this hook), `propose`
+   * to hand it to the catalog's `onProposals` instead of running it, or `{ deny }` to refuse it.
+   */
+  onCall?(
+    params: TParams,
+    context: IFlexCatalogRunContext<TScope>,
+  ): TFlexCatalogCallDecision | PromiseLike<TFlexCatalogCallDecision>;
+}
+
+export type TFlexCatalogCallDecision = 'run' | 'propose' | { deny: string };
+
+/** A call a method's `onCall` decided to propose instead of running. */
+export interface IFlexCatalogProposal {
+  /** The call's ID: the `execute` call's, or the inner call's inside `run_code`. */
+  id: string;
+  method: string;
+  params: unknown;
+}
+
+export interface IFlexCatalogProposalContext<TScope = unknown> {
+  /** The `execute` or `run_code` call that made the proposals. */
+  readonly toolCallId: string;
+  readonly abortSignal: AbortSignal;
+  readonly provider: Readonly<IFlexToolProviderContext<TScope>>;
 }
 
 /** A named type that method declarations refer to; it needs a `declaration`, a `schema`, or both. */
@@ -813,6 +850,15 @@ export interface IFlexCatalogType {
 export interface IFlexToolCatalog<TScope = unknown> {
   methods: readonly IFlexCatalogMethod<TScope>[];
   types?: readonly IFlexCatalogType[];
+  /**
+   * Receives the calls `onCall` proposed, as one batch: the proposed call of an `execute`, or every
+   * call a `run_code` program proposed, once the program finished. A failed program proposes
+   * nothing. What it returns is what the model sees for them.
+   */
+  onProposals?(
+    proposals: readonly IFlexCatalogProposal[],
+    context: IFlexCatalogProposalContext<TScope>,
+  ): unknown;
 }
 
 export interface IFlexToolCatalogOptions {
@@ -821,8 +867,79 @@ export interface IFlexToolCatalogOptions {
    * across the session, or `json-schema` with each method's self-contained JSON Schema.
    */
   declarations?: 'typescript' | 'json-schema';
+  /**
+   * Adds `run_code`, which runs a short TypeScript or JavaScript program against the catalog in a
+   * QuickJS sandbox. Needs the optional peers `quickjs-emscripten-core`,
+   * `@jitl/quickjs-ng-wasmfile-release-sync` and `amaro`.
+   */
+  codeMode?: boolean | IFlexCodeModeOptions;
+}
+
+export interface IFlexCodeModeOptions {
+  limits?: Partial<IFlexCodeModeLimits>;
 }
 
+/** The bounds of one `run_code` program. */
+export interface IFlexCodeModeLimits {
+  /** UTF-8 bytes of the program. */
+  maxSourceBytes: number;
+  /** Milliseconds the program may compute; time waiting for calls does not count. */
+  cpuMs: number;
+  /** Milliseconds the program may take in all, not counting waits for a permission decision. */
+  wallMs: number;
+  /** Bytes of the sandbox's heap. */
+  memoryBytes: number;
+  /**
+   * Bytes of the sandbox's stack. QuickJS runs on the host's stack as well, so a larger value lets
+   * deep recursion reach the host's limit first; the program then fails at its `stack` limit all
+   * the same.
+   */
+  stackBytes: number;
+  /** Catalog calls per program. */
+  maxCalls: number;
+  /** Catalog calls running at once; further calls wait for a slot. */
+  maxConcurrentCalls: number;
+  /** UTF-8 bytes of console output kept; more is cut off. */
+  maxLogBytes: number;
+}
+
+export const FLEX_CODE_MODE_TOOL_NAME = 'run_code';
+
+export const FLEX_CODE_MODE_DEFAULT_LIMITS: Readonly<IFlexCodeModeLimits> = Object.freeze({
+  maxSourceBytes: 32 * 1024,
+  cpuMs: 5_000,
+  wallMs: 60_000,
+  memoryBytes: 64 * 1024 * 1024,
+  stackBytes: 64 * 1024,
+  maxCalls: 200,
+  maxConcurrentCalls: 8,
+  maxLogBytes: 16 * 1024,
+});
+
+export const FLEX_CODE_MODE_MAXIMUM_LIMITS: Readonly<IFlexCodeModeLimits> = Object.freeze({
+  maxSourceBytes: 256 * 1024,
+  cpuMs: 60_000,
+  wallMs: 30 * 60_000,
+  memoryBytes: 512 * 1024 * 1024,
+  stackBytes: 256 * 1024,
+  maxCalls: 10_000,
+  maxConcurrentCalls: 64,
+  maxLogBytes: 256 * 1024,
+});
+
+/** What `run_code` reports to the model, as JSON, when a program does not finish. */
+export interface IFlexCodeRunFailure {
+  error: 'syntax_error' | 'program_error' | 'limit_exceeded' | 'aborted';
+  /** The limit that stopped the program, for `limit_exceeded`. */
+  limit?: 'source' | 'cpu' | 'wall' | 'memory' | 'stack' | 'calls';
+  name?: string;
+  message: string;
+  /** The program's line, when known. */
+  line?: number;
+  /** Catalog calls the program made. */
+  calls: number;
+  logs?: string;
+}
 export interface IFlexResourceToolProviderResolverContext<TScope> extends IFlexSessionGeneration {
   scopeId: string;
   scope: TScope;
@@ -1762,7 +1879,7 @@ export type TFlexModelUsageEvent = IFlexEventBase & {
   readonly type: 'model.usage';
   readonly runId?: string;
   readonly source: 'generation' | 'compaction';
-} & Readonly<TFlexModelCallUsage>;
+} & Readonly<TFlexObservedModelCallUsage>;
 
 export interface IFlexPromptQueueEvent extends IFlexEventBase {
   readonly type: 'prompt.queued' | 'prompt.started' | 'prompt.running' | 'prompt.finished';