@modelprofile.com/flexharness 12.3.0 → 12.5.0

package/ts_chat_harness_server/classes.turngate.ts

@@ -1,7 +1,12 @@
 import * as plugins from './plugins.js';
 import { FlexHarnessChatRequestError } from './errors.js';
 import { STOPPED_MESSAGE, isSameTab } from './functions.validation.js';
-import type { IFlexHarnessChatCaller, IFlexHarnessChatLimits, IFlexHarnessChatTurn } from './interfaces.js';
+import type {
+  IFlexHarnessChatCaller,
+  IFlexHarnessChatHubOptions,
+  IFlexHarnessChatLimits,
+  IFlexHarnessChatTurn,
+} from './interfaces.js';
 
 const STILL_FINISHING_MESSAGE =
   'The assistant is still finishing its previous reply. Send your message again in a moment.';
@@ -53,6 +58,7 @@ const settlesBefore = async (promiseArg: Promise<unknown>, deadlineArg: number):
 export class FlexHarnessChatTurnGate<TContext> {
   private readonly harness: TFlexHarnessChatGateHarness;
   private readonly limits: IFlexHarnessChatLimits;
+  private readonly attachmentPart: IFlexHarnessChatHubOptions<TContext>['attachmentPart'];
   private readonly bindings = new Map<string, ITurnBinding<TContext>>();
   /** serializes the admission of messages per conversation */
   private readonly locks = new Map<string, Promise<void>>();
@@ -60,9 +66,14 @@ export class FlexHarnessChatTurnGate<TContext> {
   private readonly admissions = new Set<Promise<void>>();
   private stopped = false;
 
-  constructor(harnessArg: TFlexHarnessChatGateHarness, limitsArg: IFlexHarnessChatLimits) {
+  constructor(
+    harnessArg: TFlexHarnessChatGateHarness,
+    limitsArg: IFlexHarnessChatLimits,
+    attachmentPartArg?: IFlexHarnessChatHubOptions<TContext>['attachmentPart'],
+  ) {
     this.harness = harnessArg;
     this.limits = limitsArg;
+    this.attachmentPart = attachmentPartArg;
   }
 
   public turnOf(scopeIdArg: string, sessionIdArg: string): IFlexHarnessChatTurn<TContext> | undefined {
@@ -160,10 +171,12 @@ export class FlexHarnessChatTurnGate<TContext> {
       if (!(await settlesBefore(promiseArg, deadlineArg))) throw new FlexHarnessChatRequestError(STILL_FINISHING_MESSAGE);
     };
     this.assertOpen();
+    const attachmentParts = await this.attachmentPartsOf(callerArg, target, sendArg, contextArg);
+    this.assertOpen();
     const binding = this.bindings.get(bindingKey(target.scopeId, target.sessionId));
     if (binding?.queueId) {
       for (let attempt = 0; attempt < 2; attempt++) {
-        const intent = intentOf(target, { ...sendArg, delivery: 'steer' }, binding.queueId);
+        const intent = intentOf(target, { ...sendArg, delivery: 'steer' }, { activeQueueId: binding.queueId, attachmentParts });
         if (intent.type !== 'steerPrompt') break;
         this.assertOpen();
         try {
@@ -187,7 +200,28 @@ export class FlexHarnessChatTurnGate<TContext> {
       await waitFor(binding.settled);
     }
     this.assertOpen();
-    return this.startTurn(callerArg, target, sendArg, contextArg);
+    return this.startTurn(callerArg, target, sendArg, contextArg, attachmentParts);
+  }
+
+  /** The prompt parts the host makes of the message's attachments, when it makes them. */
+  private async attachmentPartsOf(
+    callerArg: IFlexHarnessChatCaller,
+    target: ITarget,
+    sendArg: plugins.IHarnessComposerSendDetail,
+    contextArg: TContext,
+  ): Promise<plugins.TFlexPromptPart[] | undefined> {
+    if (!this.attachmentPart || sendArg.attachments.length === 0) return undefined;
+    const parts: plugins.TFlexPromptPart[] = [];
+    for (const attachment of sendArg.attachments) {
+      parts.push(await this.attachmentPart({
+        tab: { tabId: callerArg.tabId, scopeId: callerArg.scopeId },
+        scopeId: target.scopeId,
+        sessionId: target.sessionId,
+        context: contextArg,
+        attachment,
+      }));
+    }
+    return parts;
   }
 
   private async startTurn(
@@ -195,10 +229,11 @@ export class FlexHarnessChatTurnGate<TContext> {
     target: ITarget,
     sendArg: plugins.IHarnessComposerSendDetail,
     contextArg: TContext,
+    attachmentPartsArg: plugins.TFlexPromptPart[] | undefined,
   ): Promise<plugins.TFlexHarnessChatResponse<'flexHarnessChatSend'>> {
     const { scopeId, sessionId } = target;
     const key = bindingKey(scopeId, sessionId);
-    const intent = intentOf(target, { ...sendArg, delivery: 'turn' });
+    const intent = intentOf(target, { ...sendArg, delivery: 'turn' }, { attachmentParts: attachmentPartsArg });
     if (intent.type !== 'enqueuePrompt') throw new Error('A new turn must be queued.');
     let markSettled!: () => void;
     let markRunning!: () => void;
@@ -244,10 +279,13 @@ export class FlexHarnessChatTurnGate<TContext> {
 const intentOf = (
   targetArg: { scopeId: string; sessionId: string },
   sendArg: plugins.IHarnessComposerSendDetail,
-  activeQueueIdArg?: string,
+  optionsArg: { activeQueueId?: string; attachmentParts?: plugins.TFlexPromptPart[] },
 ) => {
   try {
-    return plugins.flexHarnessSendIntent(targetArg, sendArg, activeQueueIdArg ? { activeQueueId: activeQueueIdArg } : {});
+    return plugins.flexHarnessSendIntent(targetArg, sendArg, {
+      ...(optionsArg.activeQueueId ? { activeQueueId: optionsArg.activeQueueId } : {}),
+      ...(optionsArg.attachmentParts ? { attachmentParts: optionsArg.attachmentParts } : {}),
+    });
   } catch (error) {
     throw new FlexHarnessChatRequestError((error as Error)?.message || 'The message could not be read.');
   }

package/ts_chat_harness_server/index.ts

@@ -6,6 +6,7 @@ export {
   type IFlexHarnessChatErrorProjectorOptions,
 } from './functions.errorprojector.js';
 export type {
+  IFlexHarnessChatAttachmentRequest,
   IFlexHarnessChatCaller,
   IFlexHarnessChatConversation,
   IFlexHarnessChatDirectory,

package/ts_chat_harness_server/interfaces.ts

@@ -114,6 +114,18 @@ export type TFlexHarnessChatHubHarness = Pick<
   | 'redoSession'
 >;
 
+/** An attachment of a message, as the hub asks the host to turn it into the prompt part the turn sends. */
+export interface IFlexHarnessChatAttachmentRequest<TContext> {
+  /** the tab that sent the message */
+  tab: IFlexHarnessChatCaller;
+  /** the conversation the message goes to: its harness scope and session */
+  scopeId: string;
+  sessionId: string;
+  /** the context the message was sent with, as `readContext` returned it */
+  context: TContext;
+  attachment: plugins.IHarnessAttachment;
+}
+
 export interface IFlexHarnessChatHubOptions<TContext> {
   /**
    * The harness the conversations run on. The host keeps configuring it: its model resolver and
@@ -127,5 +139,14 @@ export interface IFlexHarnessChatHubOptions<TContext> {
   titleOf?(send: plugins.IHarnessComposerSendDetail): string;
   /** Where the tabs' conversations are. Defaults to the root sessions of the tab's scope. */
   directory?: IFlexHarnessChatDirectory;
+  /**
+   * Turns each attachment of a message into the prompt part the turn sends, before the message is
+   * admitted; throw `FlexHarnessChatRequestError` to refuse the message. A host whose tabs upload
+   * their files to it first gets the attachments without content, under the ids it gave the
+   * uploads, and returns references its harness's `attachmentResolver` reads. The same message
+   * may come again after a refusal. Without it, each attachment is sent with the content the
+   * composer read, and one without content is refused.
+   */
+  attachmentPart?(request: IFlexHarnessChatAttachmentRequest<TContext>): Promise<plugins.TFlexPromptPart>;
   limits?: Partial<IFlexHarnessChatLimits>;
 }

package/ts_chat_harness_server/plugins.ts

@@ -36,6 +36,7 @@ export type {
   IFlexSession,
   TFlexExternalErrorProjector,
   TFlexHarnessEvent,
+  TFlexPromptPart,
   TJsonValue,
 } from '@modelprofile.com/flexharness';
 export type {

package/ts_chat_harness_server/readme.md

@@ -53,6 +53,8 @@ The host keeps its harness, models, system prompt, tools and scopes. It authenti
 
 Without a directory, a tab's conversations are the root sessions of its scope, and the harness's session events update the tabs' lists. A host that runs each conversation in a scope of its own, or keeps records of its own, passes `directory`, an `IFlexHarnessChatDirectory`: it lists the conversations a tab reaches, finds one by its session id (one the tab does not reach is null, and reads as not found), creates one for a message without `sessionId`, renames and deletes. The caller's `scopeId` is then the tab's reach; each conversation names the harness scope its session runs in, and the hub subscribes, runs turns and calls `turnOf` in that scope. A conversation created for a message that then never starts, for instance because its session does not exist, is deleted again through the directory. With a directory, the harness's session events no longer reach the lists: the host calls `hub.publishSessions(reach, { upserts, deletedIds })` for every change, and the tabs of that reach get it as a list push.
 
+A message's attachments are sent with the content the composer read, within `maxAttachments` and `maxAttachmentChars`. A host whose tabs upload their files to it first, for files larger than a message may carry, passes `attachmentPart(request)`: before the message is admitted, the hub asks it for the prompt part of each attachment, in order, with the tab, the conversation's `scopeId` and `sessionId` and the message's context. The attachments then arrive as the composer describes them, under the ids the host gave the uploads; the host checks that the tab may send each one into that conversation and returns a reference its harness's `attachmentResolver` reads, with `sizeBytes`, so the transcript shows the file by name and size. Throw `FlexHarnessChatRequestError` to refuse the message; nothing of it is sent, and a conversation created for it is deleted again. The same message may come again after a refusal, and a steer's attachments go through the hook as a new turn's do.
+
 Refusals are `FlexHarnessChatRequestError`s with a message for the person in the tab; FlexHarness refusals keep their message. Every other error stays an internal error.
 
 ## Turns and tools in the tab

package/ts_compaction/plugins.ts

@@ -5,8 +5,16 @@ export { asSchema, streamText, wrapLanguageModel };
 export type { ModelMessage, ToolSet } from 'ai';
 export type { TSmartAiLanguageModel } from '@modelprofile.com/flexharness-models';
 
-import { AgentModelCallUsageRecorder, endModelStreamAtAbort } from '@modelprofile.com/flexharness-agent';
+import {
+  AgentModelCallUsageRecorder,
+  createAttachmentDownload,
+  endModelStreamAtAbort,
+} from '@modelprofile.com/flexharness-agent';
 
-export { AgentModelCallUsageRecorder, endModelStreamAtAbort };
+export { AgentModelCallUsageRecorder, createAttachmentDownload, endModelStreamAtAbort };
 
-export type { ProviderOptions, TAgentModelCallUsageReporter } from '@modelprofile.com/flexharness-agent';
+export type {
+  IAgentAttachmentResolver,
+  ProviderOptions,
+  TAgentModelCallUsageReporter,
+} from '@modelprofile.com/flexharness-agent';

package/ts_compaction/smartagent.compaction.ts

@@ -55,6 +55,12 @@ export interface ICompactMessagesOptions {
    * cost more than the history saves. A context compactor passes its options' `tools` through.
    */
   tools?: plugins.ToolSet;
+  /**
+   * Reads the attachments the history names by a URL of one of its schemes, so the summary call
+   * sends them as the turns do. A context compactor passes its options' `attachmentResolver`
+   * through.
+   */
+  attachmentResolver?: plugins.IAgentAttachmentResolver;
 }
 
 /**
@@ -134,6 +140,8 @@ export async function compactMessages(
         },
       })
     : model;
+  // The summary's attempts read each attachment once.
+  const download = plugins.createAttachmentDownload(options.attachmentResolver, options.abortSignal);
   // The summary is asked for in the shape of a turn, the tools offered as the turns offer them. A
   // model that calls a tool instead of answering is asked again with tool use turned off, which
   // keeps the tool definitions and so still the cached tools and instructions.
@@ -148,6 +156,7 @@ export async function compactMessages(
         ...(tools ? { tools, ...(toolChoice ? { toolChoice } : {}) } : {}),
         ...(options.providerOptions ? { providerOptions: options.providerOptions } : {}),
         abortSignal: options.abortSignal,
+        ...(download ? { experimental_download: download } : {}),
         onError: ({ error }) => {
           streamError ??= error;
         },