@modelprofile.com/flexharness 10.12.0 → 10.14.0

package/ts/utils.schemadeclarations.ts

@@ -0,0 +1,322 @@
+/**
+ * Renders JSON Schema as TypeScript declarations for the tool catalog. The output depends only on
+ * the schema, so the same schema always renders to the same bytes.
+ */
+
+export type TJsonSchemaNode = boolean | { [key: string]: unknown };
+
+export interface ISchemaDeclarationContext {
+  /** The named type a `$ref` denotes, or undefined to inline the referenced schema. */
+  namedType(ref: string): string | undefined;
+  /** The schema a local `$ref` points to. */
+  resolve(ref: string): TJsonSchemaNode | undefined;
+}
+
+export interface IRenderedDeclaration {
+  text: string;
+  /** Named types the text refers to, sorted. */
+  types: string[];
+}
+
+const identifierPattern = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
+const maxRenderDepth = 32;
+const maxInlineObjectCharacters = 100;
+const maxSafeInteger = Number.MAX_SAFE_INTEGER;
+
+export const isSchemaRecord = (value: unknown): value is { [key: string]: unknown } =>
+  typeof value === 'object' && value !== null && !Array.isArray(value);
+
+export const isIdentifier = (value: string): boolean => identifierPattern.test(value);
+
+const literal = (value: unknown): string => {
+  if (typeof value === 'string') return JSON.stringify(value);
+  if (typeof value === 'number' && Number.isFinite(value)) return String(value);
+  if (typeof value === 'boolean' || value === null) return String(value);
+  return 'unknown';
+};
+
+/**
+ * Reports whether `character` occurs outside brackets, braces, parentheses, string literals and
+ * comments.
+ */
+const hasTopLevel = (text: string, character: string): boolean => {
+  let depth = 0;
+  let quote: string | undefined;
+  for (let index = 0; index < text.length; index++) {
+    const current = text[index];
+    if (quote) {
+      if (current === '\\') index++;
+      else if (current === quote) quote = undefined;
+      continue;
+    }
+    if (current === '/' && text[index + 1] === '*') {
+      const end = text.indexOf('*/', index + 2);
+      if (end === -1) return false;
+      index = end + 1;
+      continue;
+    }
+    if (current === '"' || current === "'") quote = current;
+    else if (current === '{' || current === '[' || current === '(' || current === '<') depth++;
+    else if (current === '}' || current === ']' || current === ')' || current === '>') depth--;
+    else if (depth === 0 && current === character) return true;
+  }
+  return false;
+};
+
+const union = (parts: string[]): string => {
+  const unique = [...new Set(parts)];
+  if (unique.includes('unknown')) return 'unknown';
+  const withoutNever = unique.length > 1 ? unique.filter((part) => part !== 'never') : unique;
+  return withoutNever.join(' | ');
+};
+
+const intersection = (parts: string[]): string => {
+  const unique = [...new Set(parts)].filter((part) => part !== 'unknown');
+  if (unique.length === 0) return 'unknown';
+  return unique.map((part) => hasTopLevel(part, '|') ? `(${part})` : part).join(' & ');
+};
+
+const arrayOf = (element: string): string =>
+  hasTopLevel(element, '|') || hasTopLevel(element, '&') ? `(${element})[]` : `${element}[]`;
+
+const propertyKey = (name: string): string => isIdentifier(name) ? name : JSON.stringify(name);
+
+/** A JSDoc block at `indent`; a short single line stays on one line. */
+export const docBlock = (text: string, indent: string): string => {
+  const lines = text.replace(/\*\//g, '*\\/').split('\n').map((line) => line.trimEnd());
+  if (lines.length === 1 && indent.length + lines[0].length <= 100) return `${indent}/** ${lines[0]} */\n`;
+  return `${indent}/**\n${lines.map((line) => line ? `${indent} * ${line}` : `${indent} *`).join('\n')}\n${indent} */\n`;
+};
+
+export const countOf = (count: number, unit: string): string => `${count} ${unit}${count === 1 ? '' : 's'}`;
+
+/** Constraints a TypeScript type cannot state, in a short readable form. */
+const constraintsOf = (schema: { [key: string]: unknown }): string[] => {
+  const constraints: string[] = [];
+  const types = Array.isArray(schema.type) ? schema.type : [schema.type];
+  if (types.includes('integer')) constraints.push('integer');
+  if (typeof schema.format === 'string') constraints.push(`format ${schema.format}`);
+  const bound = (value: unknown): value is number =>
+    typeof value === 'number' && Number.isFinite(value) && Math.abs(value) !== maxSafeInteger;
+  if (bound(schema.exclusiveMinimum)) constraints.push(`> ${schema.exclusiveMinimum}`);
+  else if (bound(schema.minimum)) constraints.push(`${schema.exclusiveMinimum === true ? '>' : '>='} ${schema.minimum}`);
+  if (bound(schema.exclusiveMaximum)) constraints.push(`< ${schema.exclusiveMaximum}`);
+  else if (bound(schema.maximum)) constraints.push(`${schema.exclusiveMaximum === true ? '<' : '<='} ${schema.maximum}`);
+  if (bound(schema.multipleOf)) constraints.push(`multiple of ${schema.multipleOf}`);
+  const range = (min: unknown, max: unknown, unit: string): void => {
+    const hasMin = bound(min) && min > 0;
+    const hasMax = bound(max);
+    if (hasMin && hasMax) constraints.push(min === max ? countOf(min, unit) : `${min}-${countOf(max, unit)}`);
+    else if (hasMin) constraints.push(`at least ${countOf(min, unit)}`);
+    else if (hasMax) constraints.push(`at most ${countOf(max, unit)}`);
+  };
+  range(schema.minLength, schema.maxLength, 'character');
+  range(schema.minItems, schema.maxItems, 'item');
+  if (schema.uniqueItems === true) constraints.push('unique items');
+  if (typeof schema.pattern === 'string') constraints.push(`pattern /${schema.pattern}/`);
+  if (schema.default !== undefined) {
+    const value = JSON.stringify(schema.default);
+    if (value !== undefined && value.length <= 60) constraints.push(`default ${value}`);
+  }
+  if (schema.deprecated === true) constraints.push('deprecated');
+  return constraints;
+};
+
+/** The documentation of a property: its description and the constraints its type cannot state. */
+export const schemaDoc = (schema: TJsonSchemaNode, description?: string): string | undefined => {
+  if (!isSchemaRecord(schema)) return description?.trim() || undefined;
+  const text = (description ?? (typeof schema.description === 'string' ? schema.description : '')).trim();
+  const constraints = constraintsOf(schema);
+  if (constraints.length === 0) return text || undefined;
+  return text ? `${text} (${constraints.join(', ')})` : constraints.join(', ');
+};
+
+export class SchemaDeclarationWriter {
+  private readonly used = new Set<string>();
+  private readonly inlining = new Set<string>();
+
+  constructor(private readonly context: ISchemaDeclarationContext) {}
+
+  public get usedTypes(): string[] {
+    return [...this.used].sort();
+  }
+
+  /** The TypeScript type of `schema`, with continuation lines indented relative to `indent`. */
+  public type(schema: TJsonSchemaNode, indent = '', depth = 0, omitMemberDocs = false): string {
+    if (depth > maxRenderDepth) return 'unknown';
+    if (schema === true) return 'unknown';
+    if (schema === false) return 'never';
+    if (!isSchemaRecord(schema)) return 'unknown';
+    const base = this.baseType(schema, indent, depth, omitMemberDocs);
+    return schema.nullable === true ? union([base, 'null']) : base;
+  }
+
+  /** The members of an object schema; undefined when the schema is not a plain object type. */
+  public objectMembers(
+    schema: TJsonSchemaNode,
+  ): Array<{ name: string; doc?: string }> | undefined {
+    if (!isSchemaRecord(schema) || !this.isPlainObject(schema)) return undefined;
+    const properties = isSchemaRecord(schema.properties) ? schema.properties : {};
+    return Object.entries(properties).map(([name, property]) => {
+      const doc = schemaDoc(property as TJsonSchemaNode);
+      return { name, ...(doc === undefined ? {} : { doc }) };
+    });
+  }
+
+  /** Whether an object schema requires none of its properties. */
+  public hasNoRequiredProperties(schema: TJsonSchemaNode): boolean {
+    return isSchemaRecord(schema)
+      && this.isPlainObject(schema)
+      && !(Array.isArray(schema.required) && schema.required.length > 0);
+  }
+
+  /** `interface Name { … }` for a plain object schema, otherwise `type Name = …;`. */
+  public namedDeclaration(name: string, schema: TJsonSchemaNode, description?: string): string {
+    const doc = isSchemaRecord(schema)
+      ? (description ?? (typeof schema.description === 'string' ? schema.description : undefined))?.trim()
+      : description?.trim();
+    const prefix = doc ? docBlock(doc, '') : '';
+    if (isSchemaRecord(schema) && this.isPlainObject(schema) && schema.nullable !== true) {
+      return `${prefix}interface ${name} ${this.objectType(schema, '', 0, false)}`;
+    }
+    return `${prefix}type ${name} = ${this.type(schema, '', 0)};`;
+  }
+
+  private isPlainObject(schema: { [key: string]: unknown }): boolean {
+    if (schema.$ref !== undefined || 'const' in schema || schema.enum !== undefined) return false;
+    if (schema.anyOf !== undefined || schema.oneOf !== undefined || schema.allOf !== undefined) return false;
+    if (schema.type === 'object') return true;
+    return schema.type === undefined && isSchemaRecord(schema.properties);
+  }
+
+  private baseType(
+    schema: { [key: string]: unknown },
+    indent: string,
+    depth: number,
+    omitMemberDocs: boolean,
+  ): string {
+    if (typeof schema.$ref === 'string') return this.reference(schema.$ref, indent, depth);
+    if ('const' in schema) return literal(schema.const);
+    if (Array.isArray(schema.enum)) return schema.enum.length ? union(schema.enum.map(literal)) : 'never';
+    const alternatives = Array.isArray(schema.anyOf) ? schema.anyOf : Array.isArray(schema.oneOf) ? schema.oneOf : undefined;
+    if (alternatives?.length) {
+      return union(alternatives.map((alternative) => this.type(alternative as TJsonSchemaNode, indent, depth + 1)));
+    }
+    if (Array.isArray(schema.allOf) && schema.allOf.length) {
+      const own = { ...schema };
+      delete own.allOf;
+      const hasOwnShape = own.type !== undefined || own.properties !== undefined;
+      return intersection([
+        ...schema.allOf.map((part) => this.type(part as TJsonSchemaNode, indent, depth + 1)),
+        ...(hasOwnShape ? [this.type(own, indent, depth + 1)] : []),
+      ]);
+    }
+    const types = Array.isArray(schema.type)
+      ? schema.type.filter((type): type is string => typeof type === 'string')
+      : typeof schema.type === 'string'
+        ? [schema.type]
+        : this.inferredTypes(schema);
+    if (types.length === 0) return 'unknown';
+    return union(types.map((type) => {
+      switch (type) {
+        case 'string': return 'string';
+        case 'number':
+        case 'integer': return 'number';
+        case 'boolean': return 'boolean';
+        case 'null': return 'null';
+        case 'array': return this.arrayType(schema, indent, depth);
+        case 'object': return this.objectType(schema, indent, depth, omitMemberDocs);
+        default: return 'unknown';
+      }
+    }));
+  }
+
+  private inferredTypes(schema: { [key: string]: unknown }): string[] {
+    if (
+      schema.properties !== undefined
+      || schema.additionalProperties !== undefined
+      || schema.patternProperties !== undefined
+      || schema.required !== undefined
+    ) return ['object'];
+    if (schema.items !== undefined || schema.prefixItems !== undefined) return ['array'];
+    return [];
+  }
+
+  private reference(ref: string, indent: string, depth: number): string {
+    const name = this.context.namedType(ref);
+    if (name) {
+      this.used.add(name);
+      return name;
+    }
+    const target = this.context.resolve(ref);
+    if (target === undefined || this.inlining.has(ref)) return 'unknown';
+    this.inlining.add(ref);
+    try {
+      return this.type(target, indent, depth + 1);
+    } finally {
+      this.inlining.delete(ref);
+    }
+  }
+
+  private arrayType(schema: { [key: string]: unknown }, indent: string, depth: number): string {
+    const tuple = Array.isArray(schema.prefixItems)
+      ? schema.prefixItems
+      : Array.isArray(schema.items) ? schema.items : undefined;
+    if (tuple) {
+      const rest = Array.isArray(schema.prefixItems) ? schema.items : schema.additionalItems;
+      const members = tuple.map((item) => this.type(item as TJsonSchemaNode, indent, depth + 1));
+      if (rest !== undefined && rest !== false) {
+        members.push(`...${arrayOf(this.type(rest as TJsonSchemaNode, indent, depth + 1))}`);
+      }
+      return `[${members.join(', ')}]`;
+    }
+    if (schema.items === undefined) return 'unknown[]';
+    return arrayOf(this.type(schema.items as TJsonSchemaNode, indent, depth + 1));
+  }
+
+  private objectType(
+    schema: { [key: string]: unknown },
+    indent: string,
+    depth: number,
+    omitMemberDocs: boolean,
+  ): string {
+    const memberIndent = `${indent}  `;
+    const properties = isSchemaRecord(schema.properties) ? schema.properties : {};
+    const required = new Set(Array.isArray(schema.required)
+      ? schema.required.filter((name): name is string => typeof name === 'string')
+      : []);
+    const members: Array<{ doc?: string; text: string }> = [];
+    for (const [name, property] of Object.entries(properties)) {
+      const type = this.type(property as TJsonSchemaNode, memberIndent, depth + 1);
+      const doc = omitMemberDocs ? undefined : schemaDoc(property as TJsonSchemaNode);
+      members.push({
+        ...(doc === undefined ? {} : { doc }),
+        text: `${propertyKey(name)}${required.has(name) ? '' : '?'}: ${type}`,
+      });
+    }
+    const indexTypes: string[] = [];
+    if (isSchemaRecord(schema.patternProperties)) {
+      for (const property of Object.values(schema.patternProperties)) {
+        indexTypes.push(this.type(property as TJsonSchemaNode, memberIndent, depth + 1));
+      }
+    }
+    if (isSchemaRecord(schema.additionalProperties)) {
+      indexTypes.push(this.type(schema.additionalProperties, memberIndent, depth + 1));
+    } else if (schema.additionalProperties === true && members.length > 0) {
+      indexTypes.push('unknown');
+    }
+    if (indexTypes.length) members.push({ text: `[key: string]: ${union(indexTypes)}` });
+    if (members.length === 0) return schema.additionalProperties === false ? '{}' : 'Record<string, unknown>';
+    if (members.every((member) => member.doc === undefined && !member.text.includes('\n'))) {
+      const inline = `{ ${members.map((member) => member.text).join('; ')} }`;
+      if (inline.length <= maxInlineObjectCharacters) return inline;
+    }
+    const lines = members.map((member) =>
+      `${member.doc === undefined ? '' : docBlock(member.doc, memberIndent)}${memberIndent}${member.text};`);
+    return `{\n${lines.join('\n')}\n${indent}}`;
+  }
+}
+
+/** Indents every non-empty line of `text`. */
+export const indentLines = (text: string, indent: string): string =>
+  text.split('\n').map((line) => line ? `${indent}${line}` : line).join('\n');

package/ts/utils.tooloutput.ts

@@ -25,6 +25,15 @@ async function* normalizeAsyncIterable(
   }
 }
 
+/** A tool output's final value: the value itself, or the last value an async iterable yields. */
+export async function settleToolOutput(output: unknown): Promise<unknown> {
+  const value = await output;
+  if (!isAsyncIterable(value)) return value;
+  let last: unknown;
+  for await (const item of value) last = item;
+  return last;
+}
+
 export function wrapToolSet(
   tools: TFlexAgentToolSet,
   limits: Required<NonNullable<IFlexHarnessOptions<unknown>['toolOutputLimits']>>,

package/ts_compaction/plugins.ts

@@ -1,8 +1,8 @@
-import { streamText, wrapLanguageModel } from 'ai';
+import { asSchema, streamText, wrapLanguageModel } from 'ai';
 
-export { streamText, wrapLanguageModel };
+export { asSchema, streamText, wrapLanguageModel };
 
-export type { ModelMessage } from 'ai';
+export type { ModelMessage, ToolSet } from 'ai';
 export type { TSmartAiLanguageModel } from '@modelprofile.com/flexharness-models';
 
 import { AgentModelCallUsageRecorder } from '@modelprofile.com/flexharness-agent';

package/ts_compaction/smartagent.compaction.ts

@@ -25,6 +25,9 @@ When constructing the summary, try to stick to this template:
 /** The instructions of a summary call whose caller names none of the conversation's own. */
 const COMPACTION_INSTRUCTIONS = 'You summarize a conversation so that another agent can continue its work.';
 
+/** Added to the summary request when it carries the conversation's tools. */
+const NO_TOOL_CALLS = 'Do not call any tool. Reply with the summary text only.';
+
 export interface ICompactMessagesOptions {
   abortSignal?: AbortSignal;
   /**
@@ -45,13 +48,51 @@ export interface ICompactMessagesOptions {
    * options' `providerOptions` through.
    */
   providerOptions?: plugins.ProviderOptions;
+  /**
+   * The tools the conversation's turns are sent with. The summary call carries their definitions,
+   * none of their code, so its request repeats the turns' prefix and reads it from the provider's
+   * prompt cache, unless the definitions outweigh the history nine times, when reading them would
+   * cost more than the history saves. A context compactor passes its options' `tools` through.
+   */
+  tools?: plugins.ToolSet;
+}
+
+/**
+ * Whether the summary request should carry the turns' tool definitions. A cached prefix is read at
+ * about a tenth of the input price (Anthropic and OpenAI), so repeating the turns' prefix costs about
+ * 0.1 x (tools + history), while a request without the tools reads nothing and pays the history in
+ * full: the tools pay their way while they are less than nine times the history.
+ */
+async function toolsRepayTheirPrefix(tools: plugins.ToolSet, messages: plugins.ModelMessage[]): Promise<boolean> {
+  let toolBytes = 0;
+  for (const [name, definition] of Object.entries(tools)) {
+    const schema = definition.type === 'provider' ? definition.args : await plugins.asSchema(definition.inputSchema).jsonSchema;
+    toolBytes += name.length + JSON.stringify([definition.description ?? '', schema ?? null]).length;
+  }
+  return toolBytes <= 9 * JSON.stringify(messages).length;
+}
+
+/** A tool set's definitions without anything that runs: no `execute`, no input callbacks. */
+function toolDefinitions(tools: plugins.ToolSet): plugins.ToolSet {
+  return Object.fromEntries(Object.entries(tools).map(([name, definition]) => {
+    const {
+      execute: _execute,
+      onInputStart: _onInputStart,
+      onInputDelta: _onInputDelta,
+      onInputAvailable: _onInputAvailable,
+      ...rest
+    } = definition as plugins.ToolSet[string] & Record<string, unknown>;
+    return [name, rest as plugins.ToolSet[string]];
+  }));
 }
 
 /**
  * Compacts a message history into a summary.
  *
  * The summary is one streamed, instructed model call, the shape every agent turn has, so it reaches
- * a provider that serves only that shape (the ChatGPT Codex backend) the way the turns do.
+ * a provider that serves only that shape (the ChatGPT Codex backend) the way the turns do. With the
+ * turns' `tools` it also repeats their prefix, which the provider has cached; a model that answers
+ * it with a tool call instead of a summary is asked once more with tool use turned off.
  *
  * @param model    The same model used by runAgent, or a cheaper small model
  * @param messages The full message history that overflowed
@@ -63,9 +104,12 @@ export async function compactMessages(
   messages: plugins.ModelMessage[],
   options: ICompactMessagesOptions = {},
 ): Promise<plugins.ModelMessage[]> {
+  const tools = options.tools && Object.keys(options.tools).length > 0 && await toolsRepayTheirPrefix(options.tools, messages)
+    ? toolDefinitions(options.tools)
+    : undefined;
   const prompt: plugins.ModelMessage[] = [
     ...messages,
-    { role: 'user', content: COMPACTION_PROMPT },
+    { role: 'user', content: tools ? `${COMPACTION_PROMPT}\n\n${NO_TOOL_CALLS}` : COMPACTION_PROMPT },
   ];
   const reportUsage = options.reportUsage;
   const modelCalls = reportUsage
@@ -89,38 +133,50 @@ export async function compactMessages(
         },
       })
     : model;
-  let streamError: unknown;
-  try {
-    const result = plugins.streamText({
-      model: observedModel,
-      instructions: options.system ?? COMPACTION_INSTRUCTIONS,
-      messages: prompt,
-      allowSystemInMessages: true,
-      ...(options.providerOptions ? { providerOptions: options.providerOptions } : {}),
-      abortSignal: options.abortSignal,
-      onError: ({ error }) => {
-        streamError ??= error;
-      },
-      ...(modelCalls
-        ? {
-            onLanguageModelCallEnd: ({ modelId, usage }) => {
-              modelCalls.end(modelId, usage);
-            },
-          }
-        : {}),
-    });
-    let summary = '';
-    for await (const delta of result.textStream) summary += delta;
-    // A stream that fails delivers its error to onError and ends, and an aborted stream just ends;
-    // either way the summary is incomplete and must not replace the history.
-    if (streamError !== undefined) throw streamError;
-    options.abortSignal?.throwIfAborted();
-    modelCalls?.settle('missing');
-    return summaryMessages(summary);
-  } catch (error) {
-    modelCalls?.settle(options.abortSignal?.aborted ? 'aborted' : 'failed');
-    throw streamError ?? error;
-  }
+  // The summary is asked for in the shape of a turn, the tools offered as the turns offer them. A
+  // model that calls a tool instead of answering is asked again with tool use turned off, which
+  // keeps the tool definitions and so still the cached tools and instructions.
+  const summarize = async (toolChoice?: 'none'): Promise<{ summary: string; calledTools: boolean }> => {
+    let streamError: unknown;
+    try {
+      const result = plugins.streamText({
+        model: observedModel,
+        instructions: options.system ?? COMPACTION_INSTRUCTIONS,
+        messages: prompt,
+        allowSystemInMessages: true,
+        ...(tools ? { tools, ...(toolChoice ? { toolChoice } : {}) } : {}),
+        ...(options.providerOptions ? { providerOptions: options.providerOptions } : {}),
+        abortSignal: options.abortSignal,
+        onError: ({ error }) => {
+          streamError ??= error;
+        },
+        ...(modelCalls
+          ? {
+              onLanguageModelCallEnd: ({ modelId, usage }) => {
+                modelCalls.end(modelId, usage);
+              },
+            }
+          : {}),
+      });
+      let summary = '';
+      for await (const delta of result.textStream) summary += delta;
+      // A stream that fails delivers its error to onError and ends, and an aborted stream just ends;
+      // either way the summary is incomplete and must not replace the history.
+      if (streamError !== undefined) throw streamError;
+      options.abortSignal?.throwIfAborted();
+      modelCalls?.settle('missing');
+      return { summary, calledTools: tools !== undefined && (await result.toolCalls).length > 0 };
+    } catch (error) {
+      modelCalls?.settle(options.abortSignal?.aborted ? 'aborted' : 'failed');
+      throw streamError ?? error;
+    }
+  };
+  // A provider-executed tool, such as a web search, would run on the provider's side before the
+  // retry could prevent it: with one in the set, tool use is off from the start.
+  const providerExecuted = tools !== undefined && Object.values(tools).some((definition) => definition.type === 'provider');
+  let { summary, calledTools } = await summarize(providerExecuted ? 'none' : undefined);
+  if (calledTools && !summary.trim()) ({ summary } = await summarize('none'));
+  return summaryMessages(summary);
 }
 
 function summaryMessages(summary: string): plugins.ModelMessage[] {