@modelprofile.com/flexharness 10.23.0 → 10.24.1

package/ts/classes.flexharness.tooldispatch.ts

@@ -0,0 +1,475 @@
+import * as plugins from './plugins.js';
+import { FlexHarnessValidationError } from './errors.js';
+import {
+  FLEX_CODE_MODE_TOOL_NAME,
+  FLEX_RESULT_METHOD,
+  FLEX_TOOL_CATALOG_TOOL_NAMES,
+  type IFlexPermissionRequestInput,
+  type IFlexCatalogDescribeInput,
+  type IFlexCatalogProposal,
+  type IFlexCatalogSearchInput,
+  type IFlexToolProviderContext,
+  type IFlexUncertainToolExecution,
+  type TFlexAgentToolSet,
+  type TFlexToolExecutionReconciliation,
+  type TJsonValue,
+} from './interfaces.js';
+import { FlexToolCatalog, catalogToolInputSchema } from './classes.toolcatalog.js';
+import type { IFlexCodeHost, TFlexCodeCallOutcome } from './classes.codesandbox.js';
+import { canonicalJson, normalizeJsonValue } from './utils.json.js';
+import {
+  maxTransferMetadataBytes,
+  publicSnapshot,
+  requireTransferIdentifier,
+  truncateUtf8,
+} from './utils.transfer.js';
+import { isAsyncIterable, settleToolOutput } from './utils.tooloutput.js';
+import { validateIdentifier } from './utils.validation.js';
+import type { IActiveRun, IResultRead, IRunReadCall } from './interfaces.flexharness.js';
+import type { FlexHarnessCore } from './classes.flexharness.core.js';
+
+const notShared = Symbol('not shared');
+
+/**
+ * Tool call dispatch: read-only classification, call tracking, reconciliation and the tool catalog.
+ */
+export class FlexHarnessToolDispatch<TScope> {
+  constructor(private readonly core: FlexHarnessCore<TScope>) {}
+
+  public async listUncertainToolExecutions(
+    scopeId: string,
+    sessionId: string,
+  ): Promise<IFlexUncertainToolExecution[]> {
+    const { state } = await this.core.resolveState(scopeId);
+    const stored = this.core.requireSession(state, sessionId);
+    this.core.assertStateAcceptingWork(state);
+    try {
+      return publicSnapshot(stored.agentSession.listUncertainToolExecutions().map((intent) => ({
+        intentId: this.core.boundedIdentifier(intent.id, 'intentId'),
+        generationId: this.core.boundedIdentifier(intent.generationId, 'generationId'),
+        toolCallId: this.core.boundedIdentifier(intent.toolCallId, 'toolCallId'),
+        toolName: this.core.boundedIdentifier(intent.toolName, 'toolName'),
+        input: normalizeJsonValue(intent.input, this.core.toolOutputLimits),
+        timestamp: new Date(intent.timestamp).toISOString(),
+      })));
+    } catch (error) {
+      throw this.core.projectOperationError(error, 'agentSession', scopeId, sessionId, 'uncertain-tools');
+    }
+  }
+
+  public async reconcileToolExecution(
+    scopeId: string,
+    sessionId: string,
+    intentId: string,
+    reconciliation: TFlexToolExecutionReconciliation,
+  ): Promise<void> {
+    validateIdentifier(intentId, 'intentId');
+    requireTransferIdentifier(intentId, 'intentId');
+    if (!reconciliation || typeof reconciliation !== 'object' || Array.isArray(reconciliation)) {
+      throw new FlexHarnessValidationError('Tool execution reconciliation must be a plain object.');
+    }
+    const { scope, state } = await this.core.resolveState(scopeId);
+    const stored = this.core.requireSession(state, sessionId);
+    this.core.assertSessionWritable(stored);
+    this.core.assertStateAcceptingWork(state);
+    let canonical: plugins.TAgentToolExecutionReconciliationOptions;
+    if (reconciliation.resolution === 'executed') {
+      const output = normalizeJsonValue(reconciliation.output, this.core.toolOutputLimits);
+      canonical = { resolution: 'executed', output, modelOutput: { type: 'json', value: output } };
+    } else if (
+      reconciliation.resolution === 'not-executed'
+      || reconciliation.resolution === 'abandoned-unknown'
+    ) {
+      if (reconciliation.error !== undefined && typeof reconciliation.error !== 'string') {
+        throw new FlexHarnessValidationError('Reconciliation error must be a string.');
+      }
+      canonical = {
+        resolution: reconciliation.resolution,
+        ...(reconciliation.error === undefined
+          ? {}
+          : { error: truncateUtf8(reconciliation.error, maxTransferMetadataBytes) }),
+      };
+    } else {
+      throw new FlexHarnessValidationError('Unknown tool execution reconciliation.');
+    }
+    try {
+      await this.core.compaction.withCompactorContext(
+        this.core.compaction.createCompactorContext(scopeId, scope.scope, state.storageKey, sessionId),
+        () => stored.agentSession.reconcileToolExecution(intentId, canonical),
+      );
+    } catch (error) {
+      throw this.core.projectOperationError(error, 'agentSession', scopeId, sessionId, 'tool-reconciliation');
+    }
+  }
+
+  /**
+   * Whether a call of the run's tools only reads: search_tools and describe_tools, an `execute` of
+   * a method declared read-only, or a call the tool handle classifies as read-only by its real name.
+   */
+  public isReadOnlyToolCall(run: IActiveRun, toolName: string, input: unknown): boolean {
+    const names = FLEX_TOOL_CATALOG_TOOL_NAMES;
+    const catalog = run.toolCatalog;
+    if (catalog) {
+      if (toolName === names.search || toolName === names.describe) return true;
+      if (toolName === names.execute) {
+        const target = catalog.callTarget(input);
+        return target !== undefined && this.isReadOnlyCatalogCall(run, catalog, target.method, target.params);
+      }
+      if (toolName === FLEX_CODE_MODE_TOOL_NAME) return false;
+    }
+    if (this.core.runs.enabledBuiltInToolNames(run).has(toolName)) return false;
+    return this.classifyHostToolCall(run, toolName, input);
+  }
+
+  /** The method's path when the prepared call only reads, for the turn's reads. */
+  private readOnlyMethod(
+    run: IActiveRun,
+    catalog: FlexToolCatalog<unknown>,
+    call: { method: string; params: unknown },
+  ): string | undefined {
+    return this.isReadOnlyCatalogCall(run, catalog, call.method, call.params) ? call.method : undefined;
+  }
+
+  /** A method whose `onCall` decides its calls is gated: its calls run in order, whatever they read. */
+  private isReadOnlyCatalogCall(
+    run: IActiveRun,
+    catalog: FlexToolCatalog<unknown>,
+    method: string,
+    params: unknown,
+  ): boolean {
+    return !catalog.decides(method)
+      && (catalog.isReadOnly(method) || this.classifyHostToolCall(run, method, params));
+  }
+
+  private classifyHostToolCall(run: IActiveRun, toolName: string, input: unknown): boolean {
+    try {
+      return run.isReadOnlyHostCall?.({ toolName, input }) === true;
+    } catch (error) {
+      throw this.core.projectExternalError(run, error, 'toolProvider');
+    }
+  }
+
+  /**
+   * Runs the run's tools through the turn's reads: a read-only call identical to an earlier read of
+   * the turn shares its output, and any other call clears them. Records the confirmation of each
+   * call that started while the model still streamed, which the permission requests naming the
+   * call wait for. The catalog's tools read and run their methods by real name themselves.
+   */
+  public trackToolCalls(run: IActiveRun, tools: TFlexAgentToolSet): TFlexAgentToolSet {
+    const catalogTools = new Set<string>(run.toolCatalog
+      ? [...Object.values(FLEX_TOOL_CATALOG_TOOL_NAMES), FLEX_CODE_MODE_TOOL_NAME]
+      : []);
+    const tracked: Record<string, unknown> = {};
+    for (const [name, tool] of Object.entries(tools)) {
+      const execute = (tool as { execute?: (input: unknown, options: unknown) => unknown }).execute;
+      if (typeof execute !== 'function') {
+        tracked[name] = tool;
+        continue;
+      }
+      tracked[name] = {
+        ...tool,
+        execute: (input: unknown, options: plugins.IAgentToolExecutionOptions & { toolCallId?: string }) => {
+          const confirmed = options?.confirmed;
+          const toolCallId = options?.toolCallId ?? '';
+          if (confirmed && toolCallId) {
+            run.unconfirmedToolCalls.set(toolCallId, confirmed);
+            const forget = () => run.unconfirmedToolCalls.delete(toolCallId);
+            confirmed.then(forget, forget);
+          }
+          const executeTool = () => execute.call(tool, input, options);
+          if (catalogTools.has(name)) return executeTool();
+          const readName = this.isReadOnlyToolCall(run, name, input) ? name : undefined;
+          return this.readOnce(run, toolCallId, readName, input, confirmed, executeTool);
+        },
+      };
+    }
+    return tracked as TFlexAgentToolSet;
+  }
+
+  /**
+   * Answers a read-only call (`name` set) from an identical read earlier in the turn, sharing the
+   * output of one still running, or runs it and keeps its output for the reads after it. A failed
+   * read, and a read whose response was discarded, are not kept. Any other call clears the turn's
+   * reads before it runs.
+   */
+  private readOnce(
+    run: IActiveRun,
+    toolCallId: string,
+    name: string | undefined,
+    params: unknown,
+    confirmed: Promise<void> | undefined,
+    execute: () => unknown,
+  ): unknown {
+    if (name === undefined) {
+      run.readCalls.clear();
+      return execute();
+    }
+    const canonicalParams = canonicalJson(params);
+    if (canonicalParams === undefined) return execute();
+    const key = `${name}\n${canonicalParams}`;
+    const earlier = run.readCalls.get(key);
+    if (earlier) {
+      return earlier.output.then((output) => {
+        if (output === notShared) return execute();
+        run.deduplicatedToolCalls.set(toolCallId, earlier.toolCallId);
+        return output;
+      }, (error: unknown) => {
+        run.deduplicatedToolCalls.set(toolCallId, earlier.toolCallId);
+        throw error;
+      });
+    }
+    const result = execute();
+    const entry: IRunReadCall = {
+      toolCallId,
+      output: Promise.resolve(result).then((output) => isAsyncIterable(output) ? notShared : output),
+    };
+    run.readCalls.set(key, entry);
+    const forget = () => {
+      if (run.readCalls.get(key) === entry) run.readCalls.delete(key);
+    };
+    entry.output.catch(forget);
+    confirmed?.catch(forget);
+    return result;
+  }
+
+  /**
+   * search_tools, describe_tools and execute. `execute` validates the call and runs the method
+   * inside this tool call, so the tool wrapper bounds its output and projects its errors, and the
+   * call's part, events and permission requests carry the method's path.
+   */
+  public createCatalogTools(run: IActiveRun, catalog: FlexToolCatalog<unknown>): Record<string, unknown> {
+    const names = FLEX_TOOL_CATALOG_TOOL_NAMES;
+    const definitions = catalog.toolDefinitions();
+    const describeBytes = Math.max(1024, this.core.toolOutputLimits.maxBytes - 1024);
+    return {
+      [names.search]: plugins.tool({
+        description: definitions[names.search].description,
+        strict: false,
+        inputSchema: catalogToolInputSchema<IFlexCatalogSearchInput>(definitions[names.search]),
+        execute: async (input: IFlexCatalogSearchInput, options?: { messages?: plugins.ModelMessage[] }) =>
+          catalog.search(input, { declaredTypes: catalog.declaredTypesIn(options?.messages), maxBytes: describeBytes }),
+      }),
+      [names.describe]: plugins.tool({
+        description: definitions[names.describe].description,
+        strict: false,
+        inputSchema: catalogToolInputSchema<IFlexCatalogDescribeInput>(definitions[names.describe]),
+        execute: async (input: IFlexCatalogDescribeInput, options?: { messages?: plugins.ModelMessage[] }) =>
+          catalog.describe(input, {
+            declaredTypes: catalog.declaredTypesIn(options?.messages),
+            maxBytes: describeBytes,
+          }).output,
+      }),
+      [names.execute]: plugins.tool({
+        description: definitions[names.execute].description,
+        strict: false,
+        inputSchema: catalogToolInputSchema<{ method: string; params?: Record<string, unknown> }>(
+          definitions[names.execute],
+        ),
+        execute: async (
+          input: { method: string; params?: Record<string, unknown> },
+          options?: { toolCallId?: string; abortSignal?: AbortSignal } & plugins.IAgentToolExecutionOptions,
+        ) => {
+          const toolCallId = options?.toolCallId ?? '';
+          const abortSignal = options?.abortSignal ?? run.controller.signal;
+          const confirmed = options?.confirmed;
+          const call = await catalog.prepareCall(input.method, input.params);
+          if (!call.success) throw this.catalogCallError(run, toolCallId, call.error);
+          if (call.method === FLEX_RESULT_METHOD) {
+            const read = await this.core.results.readResult(run, call.params as IResultRead);
+            if (!read.ok) throw this.catalogCallError(run, toolCallId, read.error);
+            return read.value;
+          }
+          // A call that started early asks for permission once the response confirmed it.
+          const base = run.toolProviderContext!;
+          const provider = confirmed
+            ? Object.freeze({
+                ...base,
+                requestPermission: async (request: IFlexPermissionRequestInput) => {
+                  await confirmed;
+                  return base.requestPermission(request);
+                },
+              })
+            : base;
+          const context = { method: call.method, toolCallId, abortSignal, provider };
+          const decision = await call.decide(context);
+          if (decision === 'run') {
+            return this.readOnce(run, toolCallId, this.readOnlyMethod(run, catalog, call), call.params, confirmed,
+              () => call.run(context));
+          }
+          if (decision === 'propose') {
+            return catalog.propose(
+              [{ id: toolCallId, method: call.method, params: call.params }],
+              { toolCallId, abortSignal, provider: run.toolProviderContext! },
+            );
+          }
+          throw this.catalogCallError(run, toolCallId, { error: 'denied', method: call.method, reason: decision.deny });
+        },
+      }),
+      ...(catalog.codeMode && run.codeSandbox ? {
+        [FLEX_CODE_MODE_TOOL_NAME]: plugins.tool({
+          description: definitions[FLEX_CODE_MODE_TOOL_NAME].description,
+          strict: false,
+          inputSchema: catalogToolInputSchema<{ code: string }>(definitions[FLEX_CODE_MODE_TOOL_NAME]),
+          execute: async (input: { code: string }, options?: { toolCallId?: string; abortSignal?: AbortSignal }) =>
+            this.runCatalogProgram(run, catalog, input.code, options?.toolCallId ?? '', options?.abortSignal ?? run.controller.signal),
+        }),
+      } : {}),
+    };
+  }
+
+  /** A refused catalog call as the trusted error the model and the tool part show. */
+  private catalogCallError(run: IActiveRun, toolCallId: string, payload: unknown): Error {
+    const error = this.core.trustInternalError(new FlexHarnessValidationError(JSON.stringify(payload)));
+    if (toolCallId) run.trustedToolExecutionErrors.set(toolCallId, error);
+    return error;
+  }
+
+  /**
+   * Runs a `run_code` program. Every catalog call it makes is dispatched like an `execute`
+   * (validation, `onCall`, the method's run with its permission requests, the output limits, error
+   * projection) and recorded as a tool part of its own, named by the method, with the `run_code`
+   * call as its parent. Proposals go to `onProposals` as one batch once the program finished; a
+   * program that fails proposes nothing.
+   */
+  private async runCatalogProgram(
+    run: IActiveRun,
+    catalog: FlexToolCatalog<unknown>,
+    code: string,
+    toolCallId: string,
+    abortSignal: AbortSignal,
+  ): Promise<unknown> {
+    const proposals: IFlexCatalogProposal[] = [];
+    const base = run.toolProviderContext!;
+    // Waiting for a permission decision is the user's time, not the program's.
+    let waitedMs = 0;
+    let waiting = 0;
+    let waitingSince = 0;
+    /** The provider context of one call: its permission waits are timed, and end when the program stops. */
+    const providerFor = (signal: AbortSignal): Readonly<IFlexToolProviderContext<unknown>> => Object.freeze({
+      ...base,
+      requestPermission: async (request: IFlexPermissionRequestInput) => {
+        signal.throwIfAborted();
+        if (waiting++ === 0) waitingSince = performance.now();
+        let onAbort: (() => void) | undefined;
+        try {
+          await Promise.race([
+            base.requestPermission(request),
+            new Promise<never>((_resolve, reject) => {
+              onAbort = () => reject(signal.reason);
+              signal.addEventListener('abort', onAbort, { once: true });
+            }),
+          ]);
+        } finally {
+          if (onAbort) signal.removeEventListener('abort', onAbort);
+          if (--waiting === 0) waitedMs += performance.now() - waitingSince;
+        }
+      },
+    });
+    const stopped = (method: string) => ({ ok: false as const, error: { error: 'aborted', method } });
+    let callNumber = 0;
+    // Read-only calls run concurrently; any other call runs after every call the program made before it.
+    const order = new plugins.AgentToolCallScheduler(abortSignal);
+    const finish = (id: string, method: string, input: unknown, outcome: { output: unknown } | { error: Error }) => {
+      if ('error' in outcome) run.trustedToolExecutionErrors.set(id, outcome.error);
+      this.core.callbacks.onToolFinish(run.state, run.sessionId, 'error' in outcome
+        ? { toolCallId: id, toolName: method, input, success: false, error: outcome.error.message }
+        : { toolCallId: id, toolName: method, input, success: true, output: outcome.output });
+    };
+    const callInTurn = async (
+      id: string,
+      method: string,
+      params: unknown,
+      signal: AbortSignal,
+      classification: { readOnly: boolean } | { error: Error },
+    ): Promise<TFlexCodeCallOutcome> => {
+      const call = await catalog.prepareCall(method, params);
+      if (!call.success) {
+        if (catalog.has(method)) {
+          this.core.callbacks.onToolStart(run.state, run.sessionId, { toolCallId: id, toolName: method, input: params ?? {}, parentToolCallId: toolCallId });
+          finish(id, method, params, { error: this.catalogCallError(run, id, call.error) });
+        }
+        return { ok: false, error: call.error as unknown as TJsonValue };
+      }
+      try {
+        await order.waitForTurn(id, signal);
+      } catch {
+        return stopped(call.method);
+      }
+      if (signal.aborted) return stopped(call.method);
+      this.core.callbacks.onToolStart(run.state, run.sessionId, {
+        toolCallId: id, toolName: call.method, input: call.params, parentToolCallId: toolCallId,
+      });
+      if ('error' in classification) {
+        finish(id, call.method, call.params, { error: classification.error });
+        return { ok: false, error: { error: 'failed', method: call.method, message: classification.error.message } };
+      }
+      if (call.method === FLEX_RESULT_METHOD) {
+        const read = await this.core.results.readResult(run, call.params as IResultRead);
+        finish(id, call.method, call.params, read.ok
+          ? { output: read.value }
+          : { error: this.catalogCallError(run, id, read.error) });
+        return read;
+      }
+      const readOnly = classification.readOnly;
+      const context = { method: call.method, toolCallId: id, abortSignal: signal, provider: providerFor(signal) };
+      try {
+        const decision = await call.decide(context);
+        signal.throwIfAborted();
+        if (decision === 'propose') {
+          proposals.push({ id, method: call.method, params: call.params });
+          const value = { proposed: true, id };
+          finish(id, call.method, call.params, { output: value });
+          return { ok: true, value };
+        }
+        if (decision !== 'run') {
+          const payload = { error: 'denied' as const, method: call.method, reason: decision.deny };
+          finish(id, call.method, call.params, { error: this.catalogCallError(run, id, payload) });
+          return { ok: false, error: payload };
+        }
+        const output = this.readOnce(run, id, readOnly ? call.method : undefined, call.params, undefined,
+          () => call.run(context));
+        const value = await this.core.results.budgetOutput(run, await settleToolOutput(output));
+        finish(id, call.method, call.params, { output: value });
+        return { ok: true, value };
+      } catch (error) {
+        const projected = this.core.projectExternalError(run, error, 'toolExecution');
+        finish(id, call.method, call.params, { error: projected });
+        return { ok: false, error: { error: 'failed', method: call.method, message: projected.message } };
+      }
+    };
+    const host: IFlexCodeHost = {
+      waitedMs: () => waitedMs + (waiting > 0 ? performance.now() - waitingSince : 0),
+      call: async (method, params, signal) => {
+        if (signal.aborted) return stopped(method);
+        const id = `${toolCallId}.${++callNumber}`;
+        // The call takes its place in the program's order now, before anything is awaited.
+        let classification: { readOnly: boolean } | { error: Error };
+        try {
+          classification = {
+            readOnly: catalog.has(method) && this.isReadOnlyCatalogCall(run, catalog, method, params ?? {}),
+          };
+        } catch (error) {
+          classification = {
+            error: error instanceof Error ? error : this.core.projectExternalError(run, error, 'toolProvider'),
+          };
+        }
+        order.begin(id, 'readOnly' in classification && classification.readOnly);
+        try {
+          return await callInTurn(id, method, params, signal, classification);
+        } finally {
+          order.settle(id);
+        }
+      },
+    };
+    const result = await run.codeSandbox!.run(code, host, catalog.codeMode!, abortSignal);
+    if (!result.ok) throw this.catalogCallError(run, toolCallId, result.failure);
+    return {
+      result: result.value,
+      calls: result.calls,
+      ...(result.logs ? { logs: result.logs } : {}),
+      ...(proposals.length > 0
+        ? { proposals: await catalog.propose(proposals, { toolCallId, abortSignal, provider: base }) }
+        : {}),
+    };
+  }
+}