@modelprofile.com/flexharness 10.12.0 → 10.14.0

package/ts/classes.flexharness.ts

@@ -28,8 +28,16 @@ import {
   FLEX_REVERSION_REFERENCE_MAX_BYTES,
   FLEX_REVERSION_WORKSPACE_ID_MAX_BYTES,
   FLEX_REVERSION_WORKSPACE_LABEL_MAX_BYTES,
+  FLEX_CODE_MODE_TOOL_NAME,
   FLEX_SESSION_IMPORT_MAX_DEPTH,
+  FLEX_TOOL_CATALOG_TOOL_NAMES,
 } from './interfaces.js';
+import {
+  FlexToolCatalog,
+  assertNoCatalogCollisions,
+  catalogToolInputSchema,
+} from './classes.toolcatalog.js';
+import { FlexCodeSandbox, type IFlexCodeHost } from './classes.codesandbox.js';
 import type {
   IFlexAgentContextCompactionOptions,
   IFlexAgentContextInvocation,
@@ -112,6 +120,10 @@ import type {
   IFlexSubagentDefinition,
   IFlexSubagentProvenance,
   IFlexTerminalProjection,
+  IFlexCatalogDescribeInput,
+  IFlexCatalogProposal,
+  IFlexCatalogSearchInput,
+  IFlexToolCatalogOptions,
   IFlexToolHandle,
   IFlexToolMessagePart,
   IFlexToolProviderContext,
@@ -168,6 +180,7 @@ import {
   normalizeAgentSessionPolicy,
   normalizeBuiltInTools,
   normalizeSlashCommands,
+  normalizeToolCatalogOptions,
   normalizeSubagents,
   requireHarnessStores,
   resolveBoundedPositiveInteger,
@@ -212,7 +225,7 @@ import {
   validateSessionImportHandle,
   type INormalizedSessionImportOptions,
 } from './utils.sessionimport.js';
-import { wrapToolSet } from './utils.tooloutput.js';
+import { settleToolOutput, wrapToolSet } from './utils.tooloutput.js';
 import {
   validateCompactSessionOptions,
   validateCreateProjectTaskInput,
@@ -431,6 +444,12 @@ interface IActiveRun {
   reservedUserMessage?: IFlexMessage;
   reservedAssistantMessage?: IFlexMessage;
   modelResolution?: IFlexResolvedModel;
+  /** The tool-provider context of the run, passed to catalog methods. */
+  toolProviderContext?: Readonly<IFlexToolProviderContext<unknown>>;
+  /** The catalog methods the run offers through search_tools, describe_tools and execute. */
+  toolCatalog?: FlexToolCatalog<unknown>;
+  /** The sandbox `run_code` programs run in, loaded when the catalog has code mode. */
+  codeSandbox?: FlexCodeSandbox;
   subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
   allowedTools?: readonly string[];
   agentAllowedTools?: readonly string[];
@@ -851,6 +870,7 @@ export class FlexHarness<TScope = unknown> {
   private readonly externalErrorProjector?: TFlexExternalErrorProjector;
   private readonly subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
   private readonly builtInTools: INormalizedBuiltInTools;
+  private readonly toolCatalogOptions: Readonly<IFlexToolCatalogOptions>;
   private readonly slashCommands: ReadonlyMap<string, TRegisteredSlashCommand<TScope>>;
   private readonly maxSubagentDepth: number;
   private readonly maxSubagentCallsPerRun: number;
@@ -937,6 +957,11 @@ export class FlexHarness<TScope = unknown> {
     this.externalErrorProjector = options.externalErrorProjector;
     this.subagents = normalizeSubagents(options.subagents);
     this.builtInTools = normalizeBuiltInTools(options.builtInTools);
+    const toolCatalogOptions = normalizeToolCatalogOptions(options.toolCatalog);
+    this.toolCatalogOptions = Object.freeze({
+      declarations: toolCatalogOptions.declarations,
+      ...(toolCatalogOptions.codeMode ? { codeMode: { limits: toolCatalogOptions.codeMode } } : {}),
+    });
     this.slashCommands = normalizeSlashCommands(options.slashCommands);
     this.maxSubagentDepth = resolveBoundedPositiveInteger(
       options.maxSubagentDepth,
@@ -5823,18 +5848,38 @@ export class FlexHarness<TScope = unknown> {
       }
       signal.throwIfAborted();
       run.modelResolution = model!;
+      run.toolProviderContext = toolProviderContext as Readonly<IFlexToolProviderContext<unknown>>;
+      let catalog: FlexToolCatalog<unknown> | undefined;
+      if (toolHandle?.catalog) {
+        try {
+          catalog = await FlexToolCatalog.create(toolHandle.catalog, this.toolCatalogOptions);
+          if (catalog.codeMode) run.codeSandbox = await FlexCodeSandbox.load();
+        } catch (error) {
+          throw this.projectExternalError(run, error, 'toolProvider');
+        }
+        signal.throwIfAborted();
+      }
       let tools: TFlexAgentToolSet | undefined;
       try {
         const providedTools = toolHandle?.tools;
         const combinedTools: Record<string, unknown> = { ...(providedTools ?? {}) };
         Object.assign(combinedTools, this.createBuiltInTools(run));
+        if (catalog) assertNoCatalogCollisions(catalog, combinedTools);
         const globalNames = run.allowedTools === undefined ? undefined : new Set(run.allowedTools);
         const agentNames = run.agentAllowedTools === undefined ? undefined : new Set(run.agentAllowedTools);
+        const admitted = (name: string): boolean =>
+          (globalNames === undefined || globalNames.has(name))
+          && (agentNames === undefined || agentNames.has(name));
         const admittedTools = globalNames === undefined && agentNames === undefined
           ? combinedTools
-          : Object.fromEntries(Object.entries(combinedTools).filter(([name]) =>
-              (globalNames === undefined || globalNames.has(name))
-              && (agentNames === undefined || agentNames.has(name))));
+          : Object.fromEntries(Object.entries(combinedTools).filter(([name]) => admitted(name)));
+        const admittedCatalog = catalog && (globalNames === undefined && agentNames === undefined
+          ? catalog
+          : catalog.restrict(catalog.methods.filter(admitted)));
+        if (admittedCatalog && admittedCatalog.methods.length > 0) {
+          run.toolCatalog = admittedCatalog;
+          Object.assign(admittedTools, this.createCatalogTools(run, admittedCatalog));
+        }
         tools = Object.keys(admittedTools).length > 0
           ? wrapToolSet(
               admittedTools as TFlexAgentToolSet,
@@ -5913,6 +5958,180 @@ export class FlexHarness<TScope = unknown> {
     }
   }
 
+  /**
+   * search_tools, describe_tools and execute. `execute` validates the call and runs the method
+   * inside this tool call, so the tool wrapper bounds its output and projects its errors, and the
+   * call's part, events and permission requests carry the method's path.
+   */
+  private createCatalogTools(run: IActiveRun, catalog: FlexToolCatalog<unknown>): Record<string, unknown> {
+    const names = FLEX_TOOL_CATALOG_TOOL_NAMES;
+    const definitions = catalog.toolDefinitions();
+    const describeBytes = Math.max(1024, this.toolOutputLimits.maxBytes - 1024);
+    return {
+      [names.search]: plugins.tool({
+        description: definitions[names.search].description,
+        strict: false,
+        inputSchema: catalogToolInputSchema<IFlexCatalogSearchInput>(definitions[names.search]),
+        execute: async (input: IFlexCatalogSearchInput) => catalog.search(input),
+      }),
+      [names.describe]: plugins.tool({
+        description: definitions[names.describe].description,
+        strict: false,
+        inputSchema: catalogToolInputSchema<IFlexCatalogDescribeInput>(definitions[names.describe]),
+        execute: async (input: IFlexCatalogDescribeInput, options?: { messages?: plugins.ModelMessage[] }) =>
+          catalog.describe(input, {
+            declaredTypes: catalog.declaredTypesIn(options?.messages),
+            maxBytes: describeBytes,
+          }).output,
+      }),
+      [names.execute]: plugins.tool({
+        description: definitions[names.execute].description,
+        strict: false,
+        inputSchema: catalogToolInputSchema<{ method: string; params?: Record<string, unknown> }>(
+          definitions[names.execute],
+        ),
+        execute: async (
+          input: { method: string; params?: Record<string, unknown> },
+          options?: { toolCallId?: string; abortSignal?: AbortSignal },
+        ) => {
+          const toolCallId = options?.toolCallId ?? '';
+          const abortSignal = options?.abortSignal ?? run.controller.signal;
+          const call = await catalog.prepareCall(input.method, input.params);
+          if (!call.success) throw this.catalogCallError(run, toolCallId, call.error);
+          const context = { method: call.method, toolCallId, abortSignal, provider: run.toolProviderContext! };
+          const decision = await call.decide(context);
+          if (decision === 'run') return call.run(context);
+          if (decision === 'propose') {
+            return catalog.propose(
+              [{ id: toolCallId, method: call.method, params: call.params }],
+              { toolCallId, abortSignal, provider: run.toolProviderContext! },
+            );
+          }
+          throw this.catalogCallError(run, toolCallId, { error: 'denied', method: call.method, reason: decision.deny });
+        },
+      }),
+      ...(catalog.codeMode && run.codeSandbox ? {
+        [FLEX_CODE_MODE_TOOL_NAME]: plugins.tool({
+          description: definitions[FLEX_CODE_MODE_TOOL_NAME].description,
+          strict: false,
+          inputSchema: catalogToolInputSchema<{ code: string }>(definitions[FLEX_CODE_MODE_TOOL_NAME]),
+          execute: async (input: { code: string }, options?: { toolCallId?: string; abortSignal?: AbortSignal }) =>
+            this.runCatalogProgram(run, catalog, input.code, options?.toolCallId ?? '', options?.abortSignal ?? run.controller.signal),
+        }),
+      } : {}),
+    };
+  }
+
+  /** A refused catalog call as the trusted error the model and the tool part show. */
+  private catalogCallError(run: IActiveRun, toolCallId: string, payload: unknown): Error {
+    const error = this.trustInternalError(new FlexHarnessValidationError(JSON.stringify(payload)));
+    if (toolCallId) run.trustedToolExecutionErrors.set(toolCallId, error);
+    return error;
+  }
+
+  /**
+   * Runs a `run_code` program. Every catalog call it makes is dispatched like an `execute`
+   * (validation, `onCall`, the method's run with its permission requests, the output limits, error
+   * projection) and recorded as a tool part of its own, named by the method, with the `run_code`
+   * call as its parent. Proposals go to `onProposals` as one batch once the program finished; a
+   * program that fails proposes nothing.
+   */
+  private async runCatalogProgram(
+    run: IActiveRun,
+    catalog: FlexToolCatalog<unknown>,
+    code: string,
+    toolCallId: string,
+    abortSignal: AbortSignal,
+  ): Promise<unknown> {
+    const proposals: IFlexCatalogProposal[] = [];
+    const base = run.toolProviderContext!;
+    // Waiting for a permission decision is the user's time, not the program's.
+    let waitedMs = 0;
+    let waiting = 0;
+    let waitingSince = 0;
+    /** The provider context of one call: its permission waits are timed, and end when the program stops. */
+    const providerFor = (signal: AbortSignal): Readonly<IFlexToolProviderContext<unknown>> => Object.freeze({
+      ...base,
+      requestPermission: async (request: IFlexPermissionRequestInput) => {
+        signal.throwIfAborted();
+        if (waiting++ === 0) waitingSince = performance.now();
+        let onAbort: (() => void) | undefined;
+        try {
+          await Promise.race([
+            base.requestPermission(request),
+            new Promise<never>((_resolve, reject) => {
+              onAbort = () => reject(signal.reason);
+              signal.addEventListener('abort', onAbort, { once: true });
+            }),
+          ]);
+        } finally {
+          if (onAbort) signal.removeEventListener('abort', onAbort);
+          if (--waiting === 0) waitedMs += performance.now() - waitingSince;
+        }
+      },
+    });
+    const stopped = (method: string) => ({ ok: false as const, error: { error: 'aborted', method } });
+    let callNumber = 0;
+    const finish = (id: string, method: string, input: unknown, outcome: { output: unknown } | { error: Error }) => {
+      if ('error' in outcome) run.trustedToolExecutionErrors.set(id, outcome.error);
+      this.onToolFinish(run.state, run.sessionId, 'error' in outcome
+        ? { toolCallId: id, toolName: method, input, success: false, error: outcome.error.message }
+        : { toolCallId: id, toolName: method, input, success: true, output: outcome.output });
+    };
+    const host: IFlexCodeHost = {
+      waitedMs: () => waitedMs + (waiting > 0 ? performance.now() - waitingSince : 0),
+      call: async (method, params, signal) => {
+        if (signal.aborted) return stopped(method);
+        const id = `${toolCallId}.${++callNumber}`;
+        const call = await catalog.prepareCall(method, params);
+        if (!call.success) {
+          if (catalog.has(method)) {
+            this.onToolStart(run.state, run.sessionId, { toolCallId: id, toolName: method, input: params ?? {}, parentToolCallId: toolCallId });
+            finish(id, method, params, { error: this.catalogCallError(run, id, call.error) });
+          }
+          return { ok: false, error: call.error as unknown as TJsonValue };
+        }
+        if (signal.aborted) return stopped(call.method);
+        this.onToolStart(run.state, run.sessionId, {
+          toolCallId: id, toolName: call.method, input: call.params, parentToolCallId: toolCallId,
+        });
+        const context = { method: call.method, toolCallId: id, abortSignal: signal, provider: providerFor(signal) };
+        try {
+          const decision = await call.decide(context);
+          signal.throwIfAborted();
+          if (decision === 'propose') {
+            proposals.push({ id, method: call.method, params: call.params });
+            const value = { proposed: true, id };
+            finish(id, call.method, call.params, { output: value });
+            return { ok: true, value };
+          }
+          if (decision !== 'run') {
+            const payload = { error: 'denied' as const, method: call.method, reason: decision.deny };
+            finish(id, call.method, call.params, { error: this.catalogCallError(run, id, payload) });
+            return { ok: false, error: payload };
+          }
+          const value = normalizeJsonValue(await settleToolOutput(call.run(context)), this.toolOutputLimits);
+          finish(id, call.method, call.params, { output: value });
+          return { ok: true, value };
+        } catch (error) {
+          const projected = this.projectExternalError(run, error, 'toolExecution');
+          finish(id, call.method, call.params, { error: projected });
+          return { ok: false, error: { error: 'failed', method: call.method, message: projected.message } };
+        }
+      },
+    };
+    const result = await run.codeSandbox!.run(code, host, catalog.codeMode!, abortSignal);
+    if (!result.ok) throw this.catalogCallError(run, toolCallId, result.failure);
+    return {
+      result: result.value,
+      calls: result.calls,
+      ...(result.logs ? { logs: result.logs } : {}),
+      ...(proposals.length > 0
+        ? { proposals: await catalog.propose(proposals, { toolCallId, abortSignal, provider: base }) }
+        : {}),
+    };
+  }
+
   private createBuiltInTools(run: IActiveRun): Record<string, unknown> {
     const tools: Record<string, unknown> = {};
     if (run.subagents.size > 0 && (run.stored.session.depth ?? 0) < this.maxSubagentDepth) {
@@ -8417,20 +8636,27 @@ export class FlexHarness<TScope = unknown> {
   private onToolStart(
     state: IStorageState,
     sessionId: string,
-    event: { toolCallId: string; toolName: string; input: unknown },
+    event: { toolCallId: string; toolName: string; input: unknown; parentToolCallId?: string },
   ): void {
     const run = this.callbackRun(state, sessionId);
     if (!run || run.toolPartIds.has(event.toolCallId)) return;
     try {
-      const input = normalizeJsonValue(event.input, this.toolOutputLimits);
+      // An execute call is the call of the catalog method it names.
+      const target = event.toolName === FLEX_TOOL_CATALOG_TOOL_NAMES.execute
+        ? run.toolCatalog?.callTarget(event.input)
+        : undefined;
+      const input = normalizeJsonValue(target ? target.params : event.input, this.toolOutputLimits);
       if (!this.reserveCallbackCapacity(run, 1, jsonBytes(input), 1)) return;
       const part: IFlexToolMessagePart = {
         partId: plugins.crypto.randomUUID(),
         type: 'tool',
         toolCallId: this.boundedIdentifier(event.toolCallId, 'toolCallId'),
-        toolName: this.boundedIdentifier(event.toolName, 'toolName'),
+        toolName: this.boundedIdentifier(target ? target.method : event.toolName, 'toolName'),
         status: 'running',
         input,
+        ...(event.parentToolCallId === undefined
+          ? {}
+          : { parentToolCallId: this.boundedIdentifier(event.parentToolCallId, 'parentToolCallId') }),
         startedAt: new Date().toISOString(),
       };
       run.toolPartIds.set(event.toolCallId, part.partId);
@@ -9129,9 +9355,14 @@ export class FlexHarness<TScope = unknown> {
       }
       const tools = handle.tools;
       this.assertNoBuiltInToolCollisions(run, tools);
+      const catalog = handle.catalog;
+      if (catalog !== undefined && (typeof catalog !== 'object' || catalog === null)) {
+        throw new FlexHarnessValidationError('Tool handle catalog must be an object.');
+      }
       const closeHandle = close;
       return {
         tools,
+        ...(catalog === undefined ? {} : { catalog }),
         ...(closeHandle ? { close: () => closeHandle.call(handle) } : {}),
       };
     } catch (error) {
@@ -9218,6 +9449,9 @@ export class FlexHarness<TScope = unknown> {
         );
         if (!handle) continue;
         owned.push({ handle, closed: false });
+        if (handle.catalog !== undefined) {
+          throw new FlexHarnessValidationError('Resource tool providers cannot supply a tool catalog.');
+        }
         for (const [toolName, tool] of Object.entries(handle.tools)) {
           const exposedName = createFlexResourceToolName(namespace, toolName);
           if (Object.prototype.hasOwnProperty.call(tools, exposedName)) {
@@ -9229,6 +9463,7 @@ export class FlexHarness<TScope = unknown> {
       if (owned.length === 0 && Object.keys(tools).length === 0) return undefined;
       return {
         tools: tools as TFlexAgentToolSet,
+        ...(applicationHandle?.catalog === undefined ? {} : { catalog: applicationHandle.catalog }),
         ...(owned.some((entry) => entry.handle.close) ? { close: closeOwned } : {}),
       };
     } catch (error) {