@modelprofile.com/flexharness 10.13.0 → 10.15.0

package/ts/classes.flexharness.ts

@@ -28,6 +28,7 @@ import {
   FLEX_REVERSION_REFERENCE_MAX_BYTES,
   FLEX_REVERSION_WORKSPACE_ID_MAX_BYTES,
   FLEX_REVERSION_WORKSPACE_LABEL_MAX_BYTES,
+  FLEX_CODE_MODE_TOOL_NAME,
   FLEX_SESSION_IMPORT_MAX_DEPTH,
   FLEX_TOOL_CATALOG_TOOL_NAMES,
 } from './interfaces.js';
@@ -36,6 +37,7 @@ import {
   assertNoCatalogCollisions,
   catalogToolInputSchema,
 } from './classes.toolcatalog.js';
+import { FlexCodeSandbox, type IFlexCodeHost } from './classes.codesandbox.js';
 import type {
   IFlexAgentContextCompactionOptions,
   IFlexAgentContextInvocation,
@@ -119,6 +121,7 @@ import type {
   IFlexSubagentProvenance,
   IFlexTerminalProjection,
   IFlexCatalogDescribeInput,
+  IFlexCatalogProposal,
   IFlexCatalogSearchInput,
   IFlexToolCatalogOptions,
   IFlexToolHandle,
@@ -222,7 +225,7 @@ import {
   validateSessionImportHandle,
   type INormalizedSessionImportOptions,
 } from './utils.sessionimport.js';
-import { wrapToolSet } from './utils.tooloutput.js';
+import { settleToolOutput, wrapToolSet } from './utils.tooloutput.js';
 import {
   validateCompactSessionOptions,
   validateCreateProjectTaskInput,
@@ -445,6 +448,8 @@ interface IActiveRun {
   toolProviderContext?: Readonly<IFlexToolProviderContext<unknown>>;
   /** The catalog methods the run offers through search_tools, describe_tools and execute. */
   toolCatalog?: FlexToolCatalog<unknown>;
+  /** The sandbox `run_code` programs run in, loaded when the catalog has code mode. */
+  codeSandbox?: FlexCodeSandbox;
   subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
   allowedTools?: readonly string[];
   agentAllowedTools?: readonly string[];
@@ -865,7 +870,7 @@ export class FlexHarness<TScope = unknown> {
   private readonly externalErrorProjector?: TFlexExternalErrorProjector;
   private readonly subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
   private readonly builtInTools: INormalizedBuiltInTools;
-  private readonly toolCatalogOptions: Readonly<Required<IFlexToolCatalogOptions>>;
+  private readonly toolCatalogOptions: Readonly<IFlexToolCatalogOptions>;
   private readonly slashCommands: ReadonlyMap<string, TRegisteredSlashCommand<TScope>>;
   private readonly maxSubagentDepth: number;
   private readonly maxSubagentCallsPerRun: number;
@@ -952,7 +957,13 @@ export class FlexHarness<TScope = unknown> {
     this.externalErrorProjector = options.externalErrorProjector;
     this.subagents = normalizeSubagents(options.subagents);
     this.builtInTools = normalizeBuiltInTools(options.builtInTools);
-    this.toolCatalogOptions = normalizeToolCatalogOptions(options.toolCatalog);
+    const toolCatalogOptions = normalizeToolCatalogOptions(options.toolCatalog);
+    this.toolCatalogOptions = Object.freeze({
+      declarations: toolCatalogOptions.declarations,
+      outputTypeDepth: toolCatalogOptions.outputTypeDepth,
+      describeTopMatch: toolCatalogOptions.describeTopMatch,
+      ...(toolCatalogOptions.codeMode ? { codeMode: { limits: toolCatalogOptions.codeMode } } : {}),
+    });
     this.slashCommands = normalizeSlashCommands(options.slashCommands);
     this.maxSubagentDepth = resolveBoundedPositiveInteger(
       options.maxSubagentDepth,
@@ -5844,6 +5855,7 @@ export class FlexHarness<TScope = unknown> {
       if (toolHandle?.catalog) {
         try {
           catalog = await FlexToolCatalog.create(toolHandle.catalog, this.toolCatalogOptions);
+          if (catalog.codeMode) run.codeSandbox = await FlexCodeSandbox.load();
         } catch (error) {
           throw this.projectExternalError(run, error, 'toolProvider');
         }
@@ -5962,7 +5974,8 @@ export class FlexHarness<TScope = unknown> {
         description: definitions[names.search].description,
         strict: false,
         inputSchema: catalogToolInputSchema<IFlexCatalogSearchInput>(definitions[names.search]),
-        execute: async (input: IFlexCatalogSearchInput) => catalog.search(input),
+        execute: async (input: IFlexCatalogSearchInput, options?: { messages?: plugins.ModelMessage[] }) =>
+          catalog.search(input, { declaredTypes: catalog.declaredTypesIn(options?.messages), maxBytes: describeBytes }),
       }),
       [names.describe]: plugins.tool({
         description: definitions[names.describe].description,
@@ -5984,20 +5997,141 @@ export class FlexHarness<TScope = unknown> {
           input: { method: string; params?: Record<string, unknown> },
           options?: { toolCallId?: string; abortSignal?: AbortSignal },
         ) => {
+          const toolCallId = options?.toolCallId ?? '';
+          const abortSignal = options?.abortSignal ?? run.controller.signal;
           const call = await catalog.prepareCall(input.method, input.params);
-          if (!call.success) {
-            const error = this.trustInternalError(new FlexHarnessValidationError(JSON.stringify(call.error)));
-            if (options?.toolCallId) run.trustedToolExecutionErrors.set(options.toolCallId, error);
-            throw error;
+          if (!call.success) throw this.catalogCallError(run, toolCallId, call.error);
+          const context = { method: call.method, toolCallId, abortSignal, provider: run.toolProviderContext! };
+          const decision = await call.decide(context);
+          if (decision === 'run') return call.run(context);
+          if (decision === 'propose') {
+            return catalog.propose(
+              [{ id: toolCallId, method: call.method, params: call.params }],
+              { toolCallId, abortSignal, provider: run.toolProviderContext! },
+            );
           }
-          return call.run({
-            method: call.method,
-            toolCallId: options?.toolCallId ?? '',
-            abortSignal: options?.abortSignal ?? run.controller.signal,
-            provider: run.toolProviderContext!,
-          });
+          throw this.catalogCallError(run, toolCallId, { error: 'denied', method: call.method, reason: decision.deny });
         },
       }),
+      ...(catalog.codeMode && run.codeSandbox ? {
+        [FLEX_CODE_MODE_TOOL_NAME]: plugins.tool({
+          description: definitions[FLEX_CODE_MODE_TOOL_NAME].description,
+          strict: false,
+          inputSchema: catalogToolInputSchema<{ code: string }>(definitions[FLEX_CODE_MODE_TOOL_NAME]),
+          execute: async (input: { code: string }, options?: { toolCallId?: string; abortSignal?: AbortSignal }) =>
+            this.runCatalogProgram(run, catalog, input.code, options?.toolCallId ?? '', options?.abortSignal ?? run.controller.signal),
+        }),
+      } : {}),
+    };
+  }
+
+  /** A refused catalog call as the trusted error the model and the tool part show. */
+  private catalogCallError(run: IActiveRun, toolCallId: string, payload: unknown): Error {
+    const error = this.trustInternalError(new FlexHarnessValidationError(JSON.stringify(payload)));
+    if (toolCallId) run.trustedToolExecutionErrors.set(toolCallId, error);
+    return error;
+  }
+
+  /**
+   * Runs a `run_code` program. Every catalog call it makes is dispatched like an `execute`
+   * (validation, `onCall`, the method's run with its permission requests, the output limits, error
+   * projection) and recorded as a tool part of its own, named by the method, with the `run_code`
+   * call as its parent. Proposals go to `onProposals` as one batch once the program finished; a
+   * program that fails proposes nothing.
+   */
+  private async runCatalogProgram(
+    run: IActiveRun,
+    catalog: FlexToolCatalog<unknown>,
+    code: string,
+    toolCallId: string,
+    abortSignal: AbortSignal,
+  ): Promise<unknown> {
+    const proposals: IFlexCatalogProposal[] = [];
+    const base = run.toolProviderContext!;
+    // Waiting for a permission decision is the user's time, not the program's.
+    let waitedMs = 0;
+    let waiting = 0;
+    let waitingSince = 0;
+    /** The provider context of one call: its permission waits are timed, and end when the program stops. */
+    const providerFor = (signal: AbortSignal): Readonly<IFlexToolProviderContext<unknown>> => Object.freeze({
+      ...base,
+      requestPermission: async (request: IFlexPermissionRequestInput) => {
+        signal.throwIfAborted();
+        if (waiting++ === 0) waitingSince = performance.now();
+        let onAbort: (() => void) | undefined;
+        try {
+          await Promise.race([
+            base.requestPermission(request),
+            new Promise<never>((_resolve, reject) => {
+              onAbort = () => reject(signal.reason);
+              signal.addEventListener('abort', onAbort, { once: true });
+            }),
+          ]);
+        } finally {
+          if (onAbort) signal.removeEventListener('abort', onAbort);
+          if (--waiting === 0) waitedMs += performance.now() - waitingSince;
+        }
+      },
+    });
+    const stopped = (method: string) => ({ ok: false as const, error: { error: 'aborted', method } });
+    let callNumber = 0;
+    const finish = (id: string, method: string, input: unknown, outcome: { output: unknown } | { error: Error }) => {
+      if ('error' in outcome) run.trustedToolExecutionErrors.set(id, outcome.error);
+      this.onToolFinish(run.state, run.sessionId, 'error' in outcome
+        ? { toolCallId: id, toolName: method, input, success: false, error: outcome.error.message }
+        : { toolCallId: id, toolName: method, input, success: true, output: outcome.output });
+    };
+    const host: IFlexCodeHost = {
+      waitedMs: () => waitedMs + (waiting > 0 ? performance.now() - waitingSince : 0),
+      call: async (method, params, signal) => {
+        if (signal.aborted) return stopped(method);
+        const id = `${toolCallId}.${++callNumber}`;
+        const call = await catalog.prepareCall(method, params);
+        if (!call.success) {
+          if (catalog.has(method)) {
+            this.onToolStart(run.state, run.sessionId, { toolCallId: id, toolName: method, input: params ?? {}, parentToolCallId: toolCallId });
+            finish(id, method, params, { error: this.catalogCallError(run, id, call.error) });
+          }
+          return { ok: false, error: call.error as unknown as TJsonValue };
+        }
+        if (signal.aborted) return stopped(call.method);
+        this.onToolStart(run.state, run.sessionId, {
+          toolCallId: id, toolName: call.method, input: call.params, parentToolCallId: toolCallId,
+        });
+        const context = { method: call.method, toolCallId: id, abortSignal: signal, provider: providerFor(signal) };
+        try {
+          const decision = await call.decide(context);
+          signal.throwIfAborted();
+          if (decision === 'propose') {
+            proposals.push({ id, method: call.method, params: call.params });
+            const value = { proposed: true, id };
+            finish(id, call.method, call.params, { output: value });
+            return { ok: true, value };
+          }
+          if (decision !== 'run') {
+            const payload = { error: 'denied' as const, method: call.method, reason: decision.deny };
+            finish(id, call.method, call.params, { error: this.catalogCallError(run, id, payload) });
+            return { ok: false, error: payload };
+          }
+          const value = normalizeJsonValue(await settleToolOutput(call.run(context)), this.toolOutputLimits);
+          finish(id, call.method, call.params, { output: value });
+          return { ok: true, value };
+        } catch (error) {
+          const projected = this.projectExternalError(run, error, 'toolExecution');
+          finish(id, call.method, call.params, { error: projected });
+          return { ok: false, error: { error: 'failed', method: call.method, message: projected.message } };
+        }
+      },
+    };
+    const result = await run.codeSandbox!.run(code, host, catalog.codeMode!, abortSignal);
+    if (!result.ok) throw this.catalogCallError(run, toolCallId, result.failure);
+    return {
+      result: result.value,
+      calls: result.calls,
+      ...(result.logs ? { logs: result.logs } : {}),
+      ...(proposals.length > 0
+        ? { proposals: await catalog.propose(proposals, { toolCallId, abortSignal, provider: base }) }
+        : {}),
     };
   }
 
@@ -8505,7 +8639,7 @@ export class FlexHarness<TScope = unknown> {
   private onToolStart(
     state: IStorageState,
     sessionId: string,
-    event: { toolCallId: string; toolName: string; input: unknown },
+    event: { toolCallId: string; toolName: string; input: unknown; parentToolCallId?: string },
   ): void {
     const run = this.callbackRun(state, sessionId);
     if (!run || run.toolPartIds.has(event.toolCallId)) return;
@@ -8523,6 +8657,9 @@ export class FlexHarness<TScope = unknown> {
         toolName: this.boundedIdentifier(target ? target.method : event.toolName, 'toolName'),
         status: 'running',
         input,
+        ...(event.parentToolCallId === undefined
+          ? {}
+          : { parentToolCallId: this.boundedIdentifier(event.parentToolCallId, 'parentToolCallId') }),
         startedAt: new Date().toISOString(),
       };
       run.toolPartIds.set(event.toolCallId, part.partId);