@modelprofile.com/flexharness 10.24.0 → 11.0.0

package/ts/classes.flexharness.providers.ts

@@ -0,0 +1,322 @@
+import { FlexHarnessValidationError } from './errors.js';
+import type {
+  IFlexPermissionRequestInput,
+  IFlexResourceToolProviderDescriptor,
+  IFlexToolHandle,
+  IFlexToolProviderContext,
+  TFlexAgentToolSet,
+} from './interfaces.js';
+import type {
+  IDetachedCleanup,
+  IStorageState,
+  IActiveRun,
+  IResolverOutcome,
+} from './interfaces.flexharness.js';
+import {
+  requireSessionGeneration,
+  createFlexResourceIdentityDigest,
+  createFlexResourceToolNamespace,
+  createFlexResourceToolName,
+  combineErrors,
+} from './utils.flexharness.js';
+import type { FlexHarnessCore } from './classes.flexharness.core.js';
+
+const maxResourceToolProviders = 128;
+
+/** Tool provider plumbing: run tool handles, resource tool providers and detached cleanups. */
+export class FlexHarnessToolProviders<TScope> {
+  constructor(private readonly core: FlexHarnessCore<TScope>) {}
+
+  public normalizeToolHandle(run: IActiveRun, handle: IFlexToolHandle | undefined): IFlexToolHandle | undefined {
+    if (handle === undefined) return undefined;
+    let close: IFlexToolHandle['close'];
+    try {
+      close = handle.close;
+      if (close !== undefined && typeof close !== 'function') {
+        throw new FlexHarnessValidationError('Tool handle close must be a function.');
+      }
+      const tools = handle.tools;
+      this.core.runs.assertNoBuiltInToolCollisions(run, tools);
+      const catalog = handle.catalog;
+      if (catalog !== undefined && (typeof catalog !== 'object' || catalog === null)) {
+        throw new FlexHarnessValidationError('Tool handle catalog must be an object.');
+      }
+      const isReadOnlyToolCall = handle.isReadOnlyToolCall;
+      if (isReadOnlyToolCall !== undefined && typeof isReadOnlyToolCall !== 'function') {
+        throw new FlexHarnessValidationError('Tool handle isReadOnlyToolCall must be a function.');
+      }
+      const closeHandle = close;
+      return {
+        tools,
+        ...(catalog === undefined ? {} : { catalog }),
+        ...(isReadOnlyToolCall === undefined ? {} : {
+          isReadOnlyToolCall: (call: { toolName: string; input: unknown }) => isReadOnlyToolCall.call(handle, call),
+        }),
+        ...(closeHandle ? { close: () => closeHandle.call(handle) } : {}),
+      };
+    } catch (error) {
+      if (close) this.trackDetachedCleanup(run.state, run, () => close!.call(handle));
+      throw this.core.projectExternalError(run, error, 'toolProvider');
+    }
+  }
+
+  public async provideRunToolHandle(
+    run: IActiveRun,
+    context: Readonly<IFlexToolProviderContext<TScope>>,
+  ): Promise<IFlexToolHandle | undefined> {
+    if (!this.core.resourceToolProviderResolver) return this.core.toolProvider?.provideTools(context);
+
+    const resolverContext = Object.freeze({
+      scopeId: context.scopeId,
+      scope: context.scope,
+      sessionId: context.sessionId,
+      ...requireSessionGeneration(run.stored.session),
+      runId: context.runId,
+      ...(context.parentSessionId === undefined ? {} : { parentSessionId: context.parentSessionId }),
+      ...(context.agent === undefined ? {} : { agent: context.agent }),
+      signal: context.signal,
+    });
+    const resolved = await this.core.resourceToolProviderResolver.resolveResourceToolProviders(resolverContext);
+    const descriptors = this.normalizeResourceToolProviderDescriptors(resolved);
+    const owned: Array<{ handle: IFlexToolHandle; closed: boolean }> = [];
+    const tools: Record<string, unknown> = Object.create(null) as Record<string, unknown>;
+    /** The handle and own name of each exposed resource tool, for the handle's classifier. */
+    const resourceTools = new Map<string, { handle: IFlexToolHandle; toolName: string }>();
+    const closeOwned = async (): Promise<void> => {
+      const errors: unknown[] = [];
+      for (const entry of [...owned].reverse()) {
+        if (entry.closed || !entry.handle.close) continue;
+        try {
+          await entry.handle.close();
+          entry.closed = true;
+        } catch (error) {
+          errors.push(error);
+        }
+      }
+      if (errors.length > 0) throw combineErrors(errors);
+    };
+    try {
+      context.signal.throwIfAborted();
+      const applicationHandle = this.normalizeToolHandle(
+        run,
+        await this.core.toolProvider?.provideTools(context),
+      );
+      if (applicationHandle) {
+        owned.push({ handle: applicationHandle, closed: false });
+        for (const [toolName, tool] of Object.entries(applicationHandle.tools)) {
+          tools[toolName] = tool;
+        }
+      }
+      for (const descriptor of descriptors) {
+        context.signal.throwIfAborted();
+        const namespace = createFlexResourceToolNamespace(
+          descriptor.resourceId,
+          descriptor.attachmentRevision,
+        );
+        const resourceIdentity = createFlexResourceIdentityDigest(
+          descriptor.resourceId,
+          descriptor.attachmentRevision,
+        );
+        const resourceContext = Object.freeze({
+          ...context,
+          requestPermission: (request: IFlexPermissionRequestInput) => context.requestPermission({
+            ...request,
+            kind: `resource.${resourceIdentity}.${request.kind}`,
+            ...(request.rememberKey === undefined
+              ? {}
+              : { rememberKey: `resource:${resourceIdentity}:${request.rememberKey}` }),
+            metadata: {
+              resourceId: descriptor.resourceId,
+              attachmentRevision: descriptor.attachmentRevision,
+              resourceIdentity,
+              toolNamespace: namespace,
+              ...(request.metadata === undefined ? {} : { providerMetadata: request.metadata }),
+            },
+          }),
+        });
+        const handle = this.normalizeToolHandle(
+          run,
+          await descriptor.provider.provideTools(resourceContext),
+        );
+        if (!handle) continue;
+        owned.push({ handle, closed: false });
+        if (handle.catalog !== undefined) {
+          throw new FlexHarnessValidationError('Resource tool providers cannot supply a tool catalog.');
+        }
+        for (const [toolName, tool] of Object.entries(handle.tools)) {
+          const exposedName = createFlexResourceToolName(namespace, toolName);
+          if (Object.prototype.hasOwnProperty.call(tools, exposedName)) {
+            throw new FlexHarnessValidationError(`Duplicate exposed tool "${exposedName}".`);
+          }
+          tools[exposedName] = tool;
+          resourceTools.set(exposedName, { handle, toolName });
+        }
+      }
+      if (owned.length === 0 && Object.keys(tools).length === 0) return undefined;
+      const classifies = applicationHandle?.isReadOnlyToolCall !== undefined
+        || [...resourceTools.values()].some((entry) => entry.handle.isReadOnlyToolCall !== undefined);
+      return {
+        tools: tools as TFlexAgentToolSet,
+        ...(applicationHandle?.catalog === undefined ? {} : { catalog: applicationHandle.catalog }),
+        ...(classifies ? {
+          isReadOnlyToolCall: ({ toolName, input }: { toolName: string; input: unknown }) => {
+            const resource = resourceTools.get(toolName);
+            return resource
+              ? resource.handle.isReadOnlyToolCall?.({ toolName: resource.toolName, input }) === true
+              : applicationHandle?.isReadOnlyToolCall?.({ toolName, input }) === true;
+          },
+        } : {}),
+        ...(owned.some((entry) => entry.handle.close) ? { close: closeOwned } : {}),
+      };
+    } catch (error) {
+      if (owned.some((entry) => entry.handle.close && !entry.closed)) {
+        try {
+          await this.closePartialToolHandle({ tools: {}, close: closeOwned }, run);
+        } catch (cleanupError) {
+          throw combineErrors([error, cleanupError]);
+        }
+      }
+      throw error;
+    }
+  }
+
+  private normalizeResourceToolProviderDescriptors(
+    descriptors: readonly IFlexResourceToolProviderDescriptor<TScope>[],
+  ): IFlexResourceToolProviderDescriptor<TScope>[] {
+    if (!Array.isArray(descriptors) || descriptors.length > maxResourceToolProviders) {
+      throw new FlexHarnessValidationError(
+        `Resource tool providers must be an array with at most ${maxResourceToolProviders} descriptors.`,
+      );
+    }
+    const resourceIds = new Set<string>();
+    const namespaces = new Set<string>();
+    return descriptors.map((descriptor, index) => {
+      if (
+        !descriptor
+        || typeof descriptor !== 'object'
+        || Array.isArray(descriptor)
+        || ![Object.prototype, null].includes(Object.getPrototypeOf(descriptor))
+      ) {
+        throw new FlexHarnessValidationError(`Resource tool provider descriptor ${index} is invalid.`);
+      }
+      const unsupported = Object.keys(descriptor).find(
+        (key) => !['resourceId', 'attachmentRevision', 'provider'].includes(key),
+      );
+      if (unsupported) {
+        throw new FlexHarnessValidationError(
+          `Resource tool provider descriptor ${index} does not support "${unsupported}".`,
+        );
+      }
+      const resourceId = descriptor.resourceId;
+      const attachmentRevision = descriptor.attachmentRevision;
+      const provider = descriptor.provider;
+      const provideTools = provider?.provideTools;
+      const namespace = createFlexResourceToolNamespace(resourceId, attachmentRevision);
+      if (resourceIds.has(resourceId)) {
+        throw new FlexHarnessValidationError(`Duplicate resourceId "${resourceId}".`);
+      }
+      if (namespaces.has(namespace)) {
+        throw new FlexHarnessValidationError(`Duplicate resource tool namespace "${namespace}".`);
+      }
+      if (!provider || typeof provideTools !== 'function') {
+        throw new FlexHarnessValidationError(`Resource tool provider descriptor ${index} has no tool provider.`);
+      }
+      resourceIds.add(resourceId);
+      namespaces.add(namespace);
+      return {
+        resourceId,
+        attachmentRevision,
+        provider: {
+          provideTools: (context) => provideTools.call(provider, context),
+        },
+      };
+    });
+  }
+
+  public async closePartialToolHandle(handle: IFlexToolHandle, run: IActiveRun): Promise<void> {
+    try {
+      await handle.close?.();
+    } catch (error) {
+      const projected = this.core.projectExternalError(run, error, 'toolCleanup');
+      this.retainDetachedCleanup(run.state, run, () => handle.close?.(), projected);
+      throw projected;
+    }
+  }
+
+  private retainDetachedCleanup(
+    state: IStorageState,
+    run: IActiveRun,
+    cleanup: () => Promise<void> | void,
+    error: Error,
+  ): void {
+    state.detachedCleanups.add({
+      cleanup,
+      projectError: (cleanupError) => this.core.projectExternalError(run, cleanupError, 'toolCleanup'),
+      error,
+    });
+  }
+
+  public observeDetachedToolProvider(
+    outcome: Promise<IResolverOutcome<IFlexToolHandle | undefined>>,
+    run: IActiveRun,
+  ): void {
+    this.trackDetachedCleanup(run.state, run, async () => {
+      const result = await outcome;
+      if (!result.success || result.source !== 'tools' || !result.value) return;
+      const close = result.value.close;
+      if (close !== undefined && typeof close !== 'function') {
+        throw new FlexHarnessValidationError('Tool handle close must be a function.');
+      }
+      await close?.call(result.value);
+    });
+  }
+
+  private trackDetachedCleanup(
+    state: IStorageState,
+    run: IActiveRun,
+    cleanup: () => Promise<void> | void,
+  ): void {
+    const retained: IDetachedCleanup = {
+      cleanup,
+      projectError: (error) => this.core.projectExternalError(run, error, 'toolCleanup'),
+    };
+    state.detachedCleanups.add(retained);
+    void this.runDetachedCleanup(state, retained);
+  }
+
+  private runDetachedCleanup(state: IStorageState, retained: IDetachedCleanup): Promise<void> {
+    if (retained.completion) return retained.completion;
+    retained.error = undefined;
+    let completion!: Promise<void>;
+    completion = Promise.resolve()
+      .then(retained.cleanup)
+      .then(() => {
+        state.detachedCleanups.delete(retained);
+      }, (error: unknown) => {
+        retained.error = retained.projectError(error);
+      })
+      .finally(() => {
+        if (retained.completion === completion) retained.completion = undefined;
+      });
+    retained.completion = completion;
+    return completion;
+  }
+
+  public beginDetachedCleanupSettlement(state: IStorageState): Promise<void>[] {
+    return [...state.detachedCleanups].map((cleanup) =>
+      cleanup.completion ?? this.runDetachedCleanup(state, cleanup));
+  }
+
+  public async settleDetachedCleanups(
+    state: IStorageState,
+    attempts: readonly Promise<void>[],
+  ): Promise<Error[]> {
+    await Promise.all(attempts);
+    await Promise.all([...state.detachedCleanups]
+      .map((cleanup) => cleanup.completion)
+      .filter((completion): completion is Promise<void> => completion !== undefined));
+    return [...state.detachedCleanups]
+      .map((cleanup) => cleanup.error)
+      .filter((error): error is Error => error !== undefined);
+  }
+}

package/ts/classes.flexharness.results.ts

@@ -0,0 +1,98 @@
+import * as plugins from './plugins.js';
+import {
+  FLEX_RESULT_LIMITS,
+  FLEX_RESULT_METHOD,
+  type IFlexResultHandle,
+  type TJsonValue,
+} from './interfaces.js';
+import { describeShape, resultPage, selectPath, summarizeResult } from './utils.results.js';
+import { normalizeJsonValue } from './utils.json.js';
+import { jsonBytes } from './utils.transfer.js';
+import type { IActiveRun, IResultRead } from './interfaces.flexharness.js';
+import { queueOperation } from './utils.flexharness.js';
+import type { FlexHarnessCore } from './classes.flexharness.core.js';
+
+/** Budgeted tool output: oversized results stored behind result handles and read back by page. */
+export class FlexHarnessResults<TScope> {
+  /** A session's result writes, one at a time: they share the session's index of results. */
+  public readonly resultQueues = new Map<string, Promise<void>>();
+
+  constructor(private readonly core: FlexHarnessCore<TScope>) {}
+
+  /**
+   * The output the model gets: the output itself within the output budget, or, in a run with a
+   * catalog whose stores keep results, a handle to the stored output with a summary of it.
+   */
+  public async budgetOutput(run: IActiveRun, output: unknown): Promise<TJsonValue> {
+    const results = this.core.stores.results;
+    const maxBytes = this.core.toolOutputLimits.maxBytes;
+    if (!results || !run.toolCatalog) return normalizeJsonValue(output, this.core.toolOutputLimits);
+    const stored = normalizeJsonValue(output, {
+      ...this.core.toolOutputLimits,
+      maxBytes: Math.max(maxBytes, FLEX_RESULT_LIMITS.maxStoredBytes),
+    });
+    const bytes = jsonBytes(stored);
+    if (bytes <= maxBytes) return stored;
+    const handle = `res_${plugins.crypto.randomBytes(8).toString('hex')}`;
+    const read = `Too large to return whole: read it page by page with execute({ method: "${FLEX_RESULT_METHOD}", `
+      + `params: { handle: "${handle}", page: 1 } })${run.toolCatalog.codeMode
+        ? `, or in run_code with await api.results.get("${handle}", { page: 1 })`
+        : ''}; a dotted \`path\` reads a part of it.`;
+    const result: IFlexResultHandle = {
+      $flexType: 'result',
+      handle,
+      bytes,
+      pages: resultPage(stored, '', 1, this.resultPageBytes())!.pages,
+      summary: summarizeResult(stored, Math.min(FLEX_RESULT_LIMITS.summaryBytes, Math.floor(maxBytes / 4))),
+      read,
+    };
+    // A budget too small for even the handle keeps the truncated output.
+    if (jsonBytes(result) > maxBytes) return normalizeJsonValue(output, this.core.toolOutputLimits);
+    const { storageKey } = run.state;
+    await queueOperation(this.resultQueues, this.resultQueueKey(storageKey, run.sessionId), async () => {
+      await results.save(storageKey, run.sessionId, handle, stored);
+      // The session keeps its latest results; older ones are deleted.
+      const handles = await results.list(storageKey, run.sessionId);
+      for (const old of handles.slice(0, Math.max(0, handles.length - FLEX_RESULT_LIMITS.maxResultsPerSession))) {
+        await results.delete(storageKey, run.sessionId, old);
+      }
+    });
+    return result as unknown as TJsonValue;
+  }
+
+  public resultQueueKey(storageKey: string, sessionId: string): string {
+    return JSON.stringify([storageKey, sessionId]);
+  }
+
+  /** The JSON bytes of one `results.get` page: the output budget, less room for the page's fields. */
+  private resultPageBytes(): number {
+    return Math.max(64, this.core.toolOutputLimits.maxBytes - 512);
+  }
+
+  /** One page of a stored result of the run's session, or the error `results.get` reports. */
+  public async readResult(
+    run: IActiveRun,
+    params: IResultRead,
+  ): Promise<{ ok: true; value: TJsonValue } | { ok: false; error: TJsonValue }> {
+    const method = FLEX_RESULT_METHOD;
+    const stored = await this.core.stores.results!.load(run.state.storageKey, run.sessionId, params.handle);
+    if (stored === undefined) {
+      return { ok: false, error: { error: 'unknown_handle', method, handle: params.handle, hint: `A handle reads one of the latest ${FLEX_RESULT_LIMITS.maxResultsPerSession} results this session stored.` } };
+    }
+    const path = params.path ?? '';
+    const value = selectPath(stored, path);
+    if (value === undefined) {
+      return { ok: false, error: { error: 'unknown_path', method, path, hint: `The result is ${describeShape(stored)}.` } };
+    }
+    const page = params.page ?? 1;
+    const content = resultPage(value, path, page, this.resultPageBytes());
+    if (!content) {
+      const pages = resultPage(value, path, 1, this.resultPageBytes())!.pages;
+      return { ok: false, error: { error: 'unknown_page', method, page, pages } };
+    }
+    return {
+      ok: true,
+      value: { handle: params.handle, path, page, pages: content.pages, ...content.range, value: content.value },
+    };
+  }
+}