@modelprofile.com/flexharness 10.12.0 → 10.14.0

package/ts/index.ts

@@ -1,5 +1,6 @@
 export * from './classes.flexharness.js';
 export * from './classes.stores.js';
+export { FlexToolCatalog } from './classes.toolcatalog.js';
 export * from './errors.js';
 export * from './interfaces.js';
 export { normalizeJsonValue } from './utils.json.js';

package/ts/interfaces.ts

@@ -31,6 +31,8 @@ export type TFlexRetryReason = plugins.TAgentRetryReason;
  * Every call carries `provider` and `requestedModelId`; key usage caps on those two.
  */
 export type TFlexModelCallUsage = plugins.TAgentModelCallUsage;
+/** A model call's usage as harness events carry it: a reported call also states its cache hit ratio. */
+export type TFlexObservedModelCallUsage = plugins.TAgentObservedModelCallUsage;
 
 export interface IFlexTextPromptPart {
   type: 'text';
@@ -135,6 +137,8 @@ export interface IFlexToolMessagePart {
   output?: TJsonValue;
   error?: string;
   childSessionId?: string;
+  /** For a catalog call a `run_code` program made: the `run_code` call's ID. */
+  parentToolCallId?: string;
   model?: IFlexModelIdentity;
   /** ISO time the tool call began. Absent on parts recorded before part times existed. */
   startedAt?: string;
@@ -648,17 +652,294 @@ export interface IFlexToolProviderContext<TScope> extends IFlexSessionGeneration
   requestPermission(request: IFlexPermissionRequestInput): Promise<void>;
 }
 
-export interface IFlexToolHandle {
+export interface IFlexToolHandle<TScope = unknown> {
   tools: TFlexAgentToolSet;
+  /**
+   * Methods the model finds with `search_tools`, reads with `describe_tools` and calls with
+   * `execute`, instead of seeing each one as a tool on every step. Return the same object on
+   * every run: FlexHarness prepares a catalog once per object and treats it as immutable.
+   */
+  catalog?: IFlexToolCatalog<TScope>;
   close?(): Promise<void> | void;
 }
 
 export interface IFlexToolProvider<TScope> {
   provideTools(
     context: IFlexToolProviderContext<TScope>,
-  ): Promise<IFlexToolHandle | undefined> | IFlexToolHandle | undefined;
+  ): Promise<IFlexToolHandle<TScope> | undefined> | IFlexToolHandle<TScope> | undefined;
+}
+
+/** The names of the built-in catalog tools, reserved while a run has a catalog. */
+export const FLEX_TOOL_CATALOG_TOOL_NAMES = Object.freeze({
+  search: 'search_tools',
+  describe: 'describe_tools',
+  execute: 'execute',
+} as const);
+
+export const FLEX_TOOL_CATALOG_LIMITS = Object.freeze({
+  maxMethods: 4096,
+  maxTypes: 4096,
+  maxMethodPathCharacters: 160,
+  maxMethodPathSegments: 8,
+  maxIdentifierCharacters: 128,
+  maxSummaryCharacters: 120,
+  maxDescriptionBytes: 16 * 1024,
+  maxDeclarationBytes: 16 * 1024,
+  maxTags: 16,
+  maxTagCharacters: 48,
+  maxExamples: 4,
+  maxExampleBytes: 1024,
+  maxSchemaBytes: 64 * 1024,
+  defaultSearchResults: 10,
+  maxSearchResults: 50,
+  maxQueryCharacters: 512,
+  maxDescribeMethods: 16,
+  maxDescribeTypes: 32,
+  maxDescribeBytes: 64 * 1024,
+  maxParamIssues: 10,
+});
+
+export interface IFlexCatalogSearchInput {
+  /** Words to rank methods by. */
+  query?: string;
+  /** A dotted namespace to search in or, without `query`, to list. */
+  namespace?: string;
+  limit?: number;
+  offset?: number;
+}
+
+export interface IFlexCatalogDescribeInput {
+  /** Dotted method paths. */
+  methods?: readonly string[];
+  /** Named types to declare again or for the first time. */
+  types?: readonly string[];
+}
+
+export interface IFlexCatalogDescribeResult {
+  /** TypeScript text, or a JSON object in `json-schema` mode. */
+  output: TJsonValue;
+  /** The named types the output declares. */
+  declaredTypes: string[];
+}
+
+/** One way `execute`'s params miss the method's schema. */
+export interface IFlexCatalogParamIssue {
+  /** Where, such as `params.lines[0].amount`. */
+  path: string;
+  expected: string;
+  /** The given value as compact JSON, or `missing`. */
+  got: string;
 }
 
+/** What `execute` reports to the model, as JSON, when it cannot call a method. */
+export type TFlexCatalogCallError =
+  | {
+      error: 'unknown_method';
+      method: string;
+      didYouMean: string[];
+      hint: string;
+    }
+  | {
+      error: 'invalid_params';
+      method: string;
+      issues: IFlexCatalogParamIssue[];
+      /** How many more issues were found than listed. */
+      more?: number;
+      /** The lookup that returns the method's declaration. */
+      see: string;
+    }
+  | {
+      error: 'denied';
+      method: string;
+      reason: string;
+    };
+
+export interface IFlexCatalogToolDefinition {
+  description: string;
+  /** The tool's input as JSON Schema. */
+  inputSchema: plugins.JSONSchema7;
+}
+
+/**
+ * A parameter or result schema: a plain JSON Schema object, or a zod, Standard Schema or AI SDK
+ * schema. `$ref`s to `#/$defs/<Name>` or `#/definitions/<Name>` that the schema does not define name
+ * a catalog type.
+ */
+export type TFlexCatalogSchema = { readonly [keyword: string]: unknown } | plugins.FlexibleSchema<unknown>;
+
+export interface IFlexCatalogRunContext<TScope = unknown> {
+  /** The dotted path the model called. */
+  readonly method: string;
+  /**
+   * The call's ID, which its tool part and permission requests carry: the `execute` call's, or for
+   * a call inside `run_code` the inner call's, `<run_code call ID>.<n>`.
+   */
+  readonly toolCallId: string;
+  readonly abortSignal: AbortSignal;
+  /** The run's tool-provider context: scope, session, run and `requestPermission`. */
+  readonly provider: Readonly<IFlexToolProviderContext<TScope>>;
+}
+
+export interface IFlexCatalogMethod<TScope = unknown, TParams = any, TResult = unknown> {
+  /**
+   * The dotted path the model calls, such as `banking.listMovements`: up to 8 segments, each a
+   * JavaScript identifier of letters, digits and `_`.
+   */
+  method: string;
+  /** One line of at most 120 characters, shown in search results. */
+  summary: string;
+  /** The full description, shown with the declaration. */
+  description?: string;
+  /** More words `search_tools` matches. */
+  tags?: readonly string[];
+  /** Code shown as `@example` in the declaration. */
+  examples?: readonly string[];
+  /** The parameter object's schema; `execute` validates every call against it. */
+  inputSchema: TFlexCatalogSchema;
+  /** The result's schema, declared as the method's return type. */
+  outputSchema?: TFlexCatalogSchema;
+  /**
+   * Declaration text that replaces the generated one, such as a declaration built from TypeScript
+   * sources with their JSDoc: `function listMovements(params: …): Promise<…>;`, optionally after
+   * its JSDoc, with the last path segment as the function name.
+   */
+  declaration?: string;
+  /** Runs the method with validated parameters. */
+  run(
+    params: TParams,
+    context: IFlexCatalogRunContext<TScope>,
+  ): TResult | PromiseLike<TResult> | AsyncIterable<TResult>;
+  /**
+   * Decides each validated call before it runs: `run` (the default without this hook), `propose`
+   * to hand it to the catalog's `onProposals` instead of running it, or `{ deny }` to refuse it.
+   */
+  onCall?(
+    params: TParams,
+    context: IFlexCatalogRunContext<TScope>,
+  ): TFlexCatalogCallDecision | PromiseLike<TFlexCatalogCallDecision>;
+}
+
+export type TFlexCatalogCallDecision = 'run' | 'propose' | { deny: string };
+
+/** A call a method's `onCall` decided to propose instead of running. */
+export interface IFlexCatalogProposal {
+  /** The call's ID: the `execute` call's, or the inner call's inside `run_code`. */
+  id: string;
+  method: string;
+  params: unknown;
+}
+
+export interface IFlexCatalogProposalContext<TScope = unknown> {
+  /** The `execute` or `run_code` call that made the proposals. */
+  readonly toolCallId: string;
+  readonly abortSignal: AbortSignal;
+  readonly provider: Readonly<IFlexToolProviderContext<TScope>>;
+}
+
+/** A named type that method declarations refer to; it needs a `declaration`, a `schema`, or both. */
+export interface IFlexCatalogType {
+  /** A JavaScript identifier such as `IMoney`. */
+  name: string;
+  description?: string;
+  /** Validates parameters that refer to this type, and generates its declaration. */
+  schema?: TFlexCatalogSchema;
+  /** Declaration text that replaces the generated one, such as `interface IMoney { … }`. */
+  declaration?: string;
+}
+
+export interface IFlexToolCatalog<TScope = unknown> {
+  methods: readonly IFlexCatalogMethod<TScope>[];
+  types?: readonly IFlexCatalogType[];
+  /**
+   * Receives the calls `onCall` proposed, as one batch: the proposed call of an `execute`, or every
+   * call a `run_code` program proposed, once the program finished. A failed program proposes
+   * nothing. What it returns is what the model sees for them.
+   */
+  onProposals?(
+    proposals: readonly IFlexCatalogProposal[],
+    context: IFlexCatalogProposalContext<TScope>,
+  ): unknown;
+}
+
+export interface IFlexToolCatalogOptions {
+  /**
+   * What `describe_tools` returns: `typescript` (default) declarations with named types shared
+   * across the session, or `json-schema` with each method's self-contained JSON Schema.
+   */
+  declarations?: 'typescript' | 'json-schema';
+  /**
+   * Adds `run_code`, which runs a short TypeScript or JavaScript program against the catalog in a
+   * QuickJS sandbox. Needs the optional peers `quickjs-emscripten-core`,
+   * `@jitl/quickjs-ng-wasmfile-release-sync` and `amaro`.
+   */
+  codeMode?: boolean | IFlexCodeModeOptions;
+}
+
+export interface IFlexCodeModeOptions {
+  limits?: Partial<IFlexCodeModeLimits>;
+}
+
+/** The bounds of one `run_code` program. */
+export interface IFlexCodeModeLimits {
+  /** UTF-8 bytes of the program. */
+  maxSourceBytes: number;
+  /** Milliseconds the program may compute; time waiting for calls does not count. */
+  cpuMs: number;
+  /** Milliseconds the program may take in all, not counting waits for a permission decision. */
+  wallMs: number;
+  /** Bytes of the sandbox's heap. */
+  memoryBytes: number;
+  /**
+   * Bytes of the sandbox's stack. QuickJS runs on the host's stack as well, so a larger value lets
+   * deep recursion reach the host's limit first; the program then fails at its `stack` limit all
+   * the same.
+   */
+  stackBytes: number;
+  /** Catalog calls per program. */
+  maxCalls: number;
+  /** Catalog calls running at once; further calls wait for a slot. */
+  maxConcurrentCalls: number;
+  /** UTF-8 bytes of console output kept; more is cut off. */
+  maxLogBytes: number;
+}
+
+export const FLEX_CODE_MODE_TOOL_NAME = 'run_code';
+
+export const FLEX_CODE_MODE_DEFAULT_LIMITS: Readonly<IFlexCodeModeLimits> = Object.freeze({
+  maxSourceBytes: 32 * 1024,
+  cpuMs: 5_000,
+  wallMs: 60_000,
+  memoryBytes: 64 * 1024 * 1024,
+  stackBytes: 64 * 1024,
+  maxCalls: 200,
+  maxConcurrentCalls: 8,
+  maxLogBytes: 16 * 1024,
+});
+
+export const FLEX_CODE_MODE_MAXIMUM_LIMITS: Readonly<IFlexCodeModeLimits> = Object.freeze({
+  maxSourceBytes: 256 * 1024,
+  cpuMs: 60_000,
+  wallMs: 30 * 60_000,
+  memoryBytes: 512 * 1024 * 1024,
+  stackBytes: 256 * 1024,
+  maxCalls: 10_000,
+  maxConcurrentCalls: 64,
+  maxLogBytes: 256 * 1024,
+});
+
+/** What `run_code` reports to the model, as JSON, when a program does not finish. */
+export interface IFlexCodeRunFailure {
+  error: 'syntax_error' | 'program_error' | 'limit_exceeded' | 'aborted';
+  /** The limit that stopped the program, for `limit_exceeded`. */
+  limit?: 'source' | 'cpu' | 'wall' | 'memory' | 'stack' | 'calls';
+  name?: string;
+  message: string;
+  /** The program's line, when known. */
+  line?: number;
+  /** Catalog calls the program made. */
+  calls: number;
+  logs?: string;
+}
 export interface IFlexResourceToolProviderResolverContext<TScope> extends IFlexSessionGeneration {
   scopeId: string;
   scope: TScope;
@@ -1419,6 +1700,7 @@ export interface IFlexHarnessOptions<TScope> {
   externalErrorProjector?: TFlexExternalErrorProjector;
   subagents?: IFlexSubagentDefinition[];
   builtInTools?: IFlexBuiltInToolsOptions;
+  toolCatalog?: IFlexToolCatalogOptions;
   slashCommands?: readonly TFlexSlashCommandRegistration<TScope>[];
   /** Maximum child depth (root is 0). Defaults to 6; configurable from 1 through 8. */
   maxSubagentDepth?: number;
@@ -1597,7 +1879,7 @@ export type TFlexModelUsageEvent = IFlexEventBase & {
   readonly type: 'model.usage';
   readonly runId?: string;
   readonly source: 'generation' | 'compaction';
-} & Readonly<TFlexModelCallUsage>;
+} & Readonly<TFlexObservedModelCallUsage>;
 
 export interface IFlexPromptQueueEvent extends IFlexEventBase {
   readonly type: 'prompt.queued' | 'prompt.started' | 'prompt.running' | 'prompt.finished';

package/ts/plugins.ts

@@ -6,6 +6,91 @@ import * as path from 'node:path';
 
 export { AsyncLocalStorage, crypto, fs, path };
 
+// third party scope
+import { asSchema, jsonSchema } from 'ai';
+import { Validator as JsonSchemaValidator } from '@cfworker/json-schema';
+
+export { asSchema, jsonSchema, JsonSchemaValidator };
+export type { FlexibleSchema, JSONSchema7, ModelMessage, Schema } from 'ai';
+
+// optional code mode peers: QuickJS-ng compiled to WebAssembly, and amaro to strip TypeScript types
+
+/** A value inside a QuickJS context; the host disposes every handle it owns. */
+export interface IQuickJsHandle {
+  readonly alive: boolean;
+  dispose(): void;
+}
+
+export type TQuickJsResult =
+  | { value: IQuickJsHandle; error?: undefined }
+  | { error: IQuickJsHandle; value?: undefined };
+
+export interface IQuickJsDeferredPromise {
+  readonly handle: IQuickJsHandle;
+  readonly alive: boolean;
+  resolve(value?: IQuickJsHandle): void;
+  reject(value?: IQuickJsHandle): void;
+  dispose(): void;
+}
+
+export interface IQuickJsContext {
+  readonly global: IQuickJsHandle;
+  newString(value: string): IQuickJsHandle;
+  getString(handle: IQuickJsHandle): string;
+  newFunction(name: string, implementation: (...args: IQuickJsHandle[]) => IQuickJsHandle | void): IQuickJsHandle;
+  setProp(target: IQuickJsHandle, key: string, value: IQuickJsHandle): void;
+  newPromise(): IQuickJsDeferredPromise;
+  newError(message: string): IQuickJsHandle;
+  evalCode(code: string, filename?: string): TQuickJsResult;
+  resolvePromise(handle: IQuickJsHandle): Promise<TQuickJsResult>;
+  dump(handle: IQuickJsHandle): unknown;
+  dispose(): void;
+}
+
+export interface IQuickJsRuntime {
+  readonly alive: boolean;
+  setMemoryLimit(bytes: number): void;
+  setMaxStackSize(bytes: number): void;
+  setInterruptHandler(handler: () => boolean): void;
+  newContext(): IQuickJsContext;
+  /** Runs the queued promise jobs; `error` is an exception none of them caught. */
+  executePendingJobs(): { error?: IQuickJsHandle };
+  dispose(): void;
+}
+
+export interface IQuickJsModule {
+  newRuntime(): IQuickJsRuntime;
+}
+
+/** A failed strip, as amaro reports it. */
+export interface IAmaroError {
+  code?: string;
+  message?: string;
+  startLine?: number;
+}
+
+export interface ICodeSandboxEngine {
+  /** A new WebAssembly instance of QuickJS-ng, with a memory of its own. */
+  newModule(): Promise<IQuickJsModule>;
+  /** Removes TypeScript type syntax, keeping every line where it was. */
+  stripTypes(source: string): string;
+}
+
+/** Loads QuickJS and amaro, the optional peers code mode needs. */
+export const importCodeSandboxEngine = async (): Promise<ICodeSandboxEngine> => {
+  const [core, variant, amaro] = await Promise.all([
+    import('quickjs-emscripten-core'),
+    import('@jitl/quickjs-ng-wasmfile-release-sync'),
+    import('amaro'),
+  ]);
+  // The variant's typings resolve as CommonJS; at run time the default export is the variant.
+  const quickjsVariant = (variant as { default: unknown }).default as Parameters<typeof core.newQuickJSWASMModuleFromVariant>[0];
+  return {
+    newModule: async () => await core.newQuickJSWASMModuleFromVariant(quickjsVariant) as unknown as IQuickJsModule,
+    stripTypes: (source) => amaro.transformSync(source, { mode: 'strip-only' }).code,
+  };
+};
+
 // @push.rocks scope
 import {
   AgentEventStoreConflictError,
@@ -73,6 +158,7 @@ export type {
   TAgentGenerationOutcome,
   TAgentModelCallUsage,
   TAgentModelCallUsageEvent,
+  TAgentObservedModelCallUsage,
   TAgentRetryReason,
   TAgentToolCallFinishEvent,
   TAgentToolExecutionReconciliationOptions,

package/ts/readme.md

@@ -1026,7 +1026,7 @@ A `contextCompactor` states which model to compact with from the model the compa
 
 A compactor uses `resolvedModel.model` directly, and resolves `modelHint` the way its model resolver resolves a prompt's `modelHint`. Only when neither member is present, because nothing dispatched states a model, does the host choose the compaction model itself.
 
-A `context-overflow` or `context-budget` compaction also carries `system` and `providerOptions`: the instructions and the provider options (cache options applied, so `store: false` on OpenAI) the run's turns are sent with. `compactMessages()` sends its summary call with them as one streamed call, the shape of every turn, so a compactor that passes its options through reaches the provider the way the run does. Without `system` it sends its own instructions.
+A `context-overflow` or `context-budget` compaction also carries `system` and `providerOptions`: the instructions and the provider options (cache options applied, so `store: false` on OpenAI) the run's turns are sent with. A `context-budget` compaction on a provider that caches the prefix also carries the turns' `tools`. `compactMessages()` sends its summary call with them as one streamed call, the shape of every turn, with the tools' definitions but none of their code, so a compactor that passes its options through repeats the run's prefix and reads it from the provider's prompt cache; definitions that outweigh the history nine times are left out. Without `system` it sends its own instructions.
 
 ```typescript
 agentSessionPolicy: {

package/ts/utils.harnessoptions.ts

@@ -1,5 +1,7 @@
 import { FlexHarnessValidationError } from './errors.js';
 import {
+  FLEX_CODE_MODE_DEFAULT_LIMITS,
+  FLEX_CODE_MODE_MAXIMUM_LIMITS,
   FLEX_REVERSION_DEFAULT_LIMITS,
   FLEX_REVERSION_MAXIMUM_LIMITS,
 } from './interfaces.js';
@@ -12,7 +14,9 @@ import type {
   IFlexReversionLimits,
   IFlexSlashCommandHandlerRegistration,
   IFlexSlashCommandTemplateRegistration,
+  IFlexCodeModeLimits,
   IFlexSubagentDefinition,
+  IFlexToolCatalogOptions,
   TFlexSlashCommandRegistration,
 } from './interfaces.js';
 import {
@@ -156,6 +160,57 @@ export function normalizeSubagents(
   return Object.freeze(normalized);
 }
 
+export interface INormalizedToolCatalogOptions {
+  readonly declarations: 'typescript' | 'json-schema';
+  /** The limits of `run_code`; absent while code mode is off. */
+  readonly codeMode?: Readonly<IFlexCodeModeLimits>;
+}
+
+const isPlainOptions = (value: unknown): value is Record<string, unknown> =>
+  !!value
+  && typeof value === 'object'
+  && !Array.isArray(value)
+  && [Object.prototype, null].includes(Object.getPrototypeOf(value));
+
+export function normalizeToolCatalogOptions(
+  options: IFlexToolCatalogOptions | undefined,
+): INormalizedToolCatalogOptions {
+  if (options === undefined) return Object.freeze({ declarations: 'typescript' });
+  if (!isPlainOptions(options)) throw new FlexHarnessValidationError('toolCatalog must be a plain object.');
+  const unsupported = Object.keys(options).find((key) => key !== 'declarations' && key !== 'codeMode');
+  if (unsupported) throw new FlexHarnessValidationError(`toolCatalog does not support "${unsupported}".`);
+  const declarations = options.declarations ?? 'typescript';
+  if (declarations !== 'typescript' && declarations !== 'json-schema') {
+    throw new FlexHarnessValidationError('toolCatalog.declarations must be "typescript" or "json-schema".');
+  }
+  const codeMode = normalizeCodeModeLimits(options.codeMode);
+  return Object.freeze({ declarations, ...(codeMode ? { codeMode } : {}) });
+}
+
+function normalizeCodeModeLimits(codeMode: unknown): Readonly<IFlexCodeModeLimits> | undefined {
+  if (codeMode === undefined || codeMode === false) return undefined;
+  if (codeMode === true) return FLEX_CODE_MODE_DEFAULT_LIMITS;
+  if (!isPlainOptions(codeMode)) {
+    throw new FlexHarnessValidationError('toolCatalog.codeMode must be a boolean or a plain object.');
+  }
+  const unsupported = Object.keys(codeMode).find((key) => key !== 'limits');
+  if (unsupported) throw new FlexHarnessValidationError(`toolCatalog.codeMode does not support "${unsupported}".`);
+  const limits = codeMode.limits ?? {};
+  if (!isPlainOptions(limits)) throw new FlexHarnessValidationError('toolCatalog.codeMode.limits must be a plain object.');
+  const resolved: Record<string, number> = { ...FLEX_CODE_MODE_DEFAULT_LIMITS };
+  for (const [key, value] of Object.entries(limits)) {
+    if (!Object.prototype.hasOwnProperty.call(FLEX_CODE_MODE_MAXIMUM_LIMITS, key)) {
+      throw new FlexHarnessValidationError(`toolCatalog.codeMode.limits does not support "${key}".`);
+    }
+    const maximum = FLEX_CODE_MODE_MAXIMUM_LIMITS[key as keyof IFlexCodeModeLimits];
+    if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1 || value > maximum) {
+      throw new FlexHarnessValidationError(`toolCatalog.codeMode.limits.${key} must be an integer from 1 through ${maximum}.`);
+    }
+    resolved[key] = value;
+  }
+  return Object.freeze(resolved as unknown as IFlexCodeModeLimits);
+}
+
 export function normalizeBuiltInTools(
   options: IFlexHarnessOptions<unknown>['builtInTools'],
 ): INormalizedBuiltInTools {

package/ts/utils.json.ts

@@ -785,6 +785,7 @@ function validateMessage(value: unknown, path: string): IValidatedMessageIdentit
           'output',
           'error',
           'childSessionId',
+          'parentToolCallId',
           'model',
           'startedAt',
           'completedAt',
@@ -801,6 +802,7 @@ function validateMessage(value: unknown, path: string): IValidatedMessageIdentit
       }
       requireOptionalString(part.error, `${partPath}.error`);
       requireOptionalString(part.childSessionId, `${partPath}.childSessionId`);
+      requireOptionalString(part.parentToolCallId, `${partPath}.parentToolCallId`);
       if (part.model !== undefined) validateModel(part.model, `${partPath}.model`);
       requireOptionalString(part.startedAt, `${partPath}.startedAt`);
       requireOptionalString(part.completedAt, `${partPath}.completedAt`);