@modelprofile.com/flexharness 8.3.0 → 8.5.0

package/ts_compaction/smartagent.compaction.ts

@@ -24,6 +24,13 @@ When constructing the summary, try to stick to this template:
 
 export interface ICompactMessagesOptions {
   abortSignal?: AbortSignal;
+  /**
+   * Receives the usage of every model call the compaction makes, once per call: `reported` with the
+   * provider's usage, or `unreported` for a call that failed or was aborted, retried attempts
+   * included. A context compactor passes its own options through, so the calls reach the session's
+   * `onUsage`. Must not throw.
+   */
+  reportUsage?: plugins.TAgentModelCallUsageReporter;
 }
 
 /**
@@ -32,7 +39,7 @@ export interface ICompactMessagesOptions {
  *
  * @param model    The same model used by runAgent, or a cheaper small model
  * @param messages The full message history that overflowed
- * @param options  Invocation options, including cancellation
+ * @param options  Invocation options, including cancellation and usage reporting
  * @returns A minimal ModelMessage[] containing the summary as context
  */
 export async function compactMessages(
@@ -40,20 +47,60 @@ export async function compactMessages(
   messages: plugins.ModelMessage[],
   options: ICompactMessagesOptions = {},
 ): Promise<plugins.ModelMessage[]> {
-  const result = await plugins.generateText({
+  const prompt: plugins.ModelMessage[] = [
+    ...messages,
+    { role: 'user', content: COMPACTION_PROMPT },
+  ];
+  const reportUsage = options.reportUsage;
+  if (!reportUsage) {
+    const result = await plugins.generateText({
+      model,
+      abortSignal: options.abortSignal,
+      allowSystemInMessages: true,
+      messages: prompt,
+    });
+    return summaryMessages(result.text);
+  }
+  const modelCalls = new plugins.AgentModelCallUsageRecorder(model, reportUsage);
+  // Each attempt, retries included, is one model call: it starts in the middleware, which also
+  // settles an attempt that throws; a call whose response arrives ends in onLanguageModelCallEnd.
+  const observedModel = plugins.wrapLanguageModel({
     model,
-    abortSignal: options.abortSignal,
-    allowSystemInMessages: true,
-    messages: [
-      ...messages,
-      { role: 'user', content: COMPACTION_PROMPT },
-    ],
+    middleware: {
+      wrapGenerate: async ({ doGenerate, params }) => {
+        modelCalls.start();
+        try {
+          return await doGenerate();
+        } catch (error) {
+          modelCalls.settle(params.abortSignal?.aborted ? 'aborted' : 'failed');
+          throw error;
+        }
+      },
+    },
   });
+  try {
+    const result = await plugins.generateText({
+      model: observedModel,
+      abortSignal: options.abortSignal,
+      allowSystemInMessages: true,
+      messages: prompt,
+      onLanguageModelCallEnd: ({ modelId, usage }) => {
+        modelCalls.end(modelId, usage);
+      },
+    });
+    modelCalls.settle('missing');
+    return summaryMessages(result.text);
+  } catch (error) {
+    modelCalls.settle(options.abortSignal?.aborted ? 'aborted' : 'failed');
+    throw error;
+  }
+}
 
+function summaryMessages(summary: string): plugins.ModelMessage[] {
   return [
     {
       role: 'user',
-      content: `[Previous conversation summary]\n\n${result.text}\n\n[End of summary. Continue from here.]`,
+      content: `[Previous conversation summary]\n\n${summary}\n\n[End of summary. Continue from here.]`,
     },
   ];
 }

package/ts_document/index.ts

@@ -36,6 +36,14 @@ export interface IDocumentRequestOptions {
    * exceeded the limit are not rendered.
    */
   maxPages?: number;
+  /**
+   * Called once for every model call of the request, as soon as its usage is known, whatever the
+   * request's outcome: `reported` with the provider's usage, or `unreported` with the reason when
+   * the call failed, was aborted or its response carried no usage. A retried attempt is a call of
+   * its own. Nothing is reported for a request that ends before its model is called. When a request
+   * returns, its reported calls sum to its `usage`. Must not throw.
+   */
+  onUsage?: plugins.TAgentModelCallUsageReporter;
 }
 
 export interface IDocumentOptions extends IDocumentRequestOptions {
@@ -120,6 +128,7 @@ export async function extractTextFromPdfWithUsage(
     providerOptions: options.providerOptions,
     abortSignal: options.abortSignal,
     maxPages: options.maxPages,
+    onUsage: options.onUsage,
   });
 }
 
@@ -156,26 +165,61 @@ export async function analyzeDocumentsWithUsage(options: IDocumentOptions): Prom
   // Filter out empty buffers
   const validPages = imagePages.filter(page => page && page.length > 0);
 
-  const result = await plugins.generateText({
-    model: options.model,
-    instructions: options.systemMessage,
-    messages: [
-      ...(options.messageHistory ?? []),
-      {
-        role: 'user',
-        content: [
-          { type: 'text', text: options.userMessage },
-          ...validPages.map(page => ({
-            type: 'file' as const,
-            data: page,
-            mediaType: 'image/png' as const,
-          })),
-        ],
-      },
-    ],
-    providerOptions: options.providerOptions,
-    abortSignal,
-  });
+  const onUsage = options.onUsage;
+  const modelCalls = onUsage ? new plugins.AgentModelCallUsageRecorder(options.model, onUsage) : null;
+  // Each attempt, retries included, is one model call: it starts in the middleware, which also
+  // settles an attempt that throws; a call whose response arrives ends in onLanguageModelCallEnd.
+  const model = modelCalls
+    ? plugins.wrapLanguageModel({
+        model: options.model,
+        middleware: {
+          wrapGenerate: async ({ doGenerate, params }) => {
+            modelCalls.start();
+            try {
+              return await doGenerate();
+            } catch (error) {
+              modelCalls.settle(params.abortSignal?.aborted ? 'aborted' : 'failed');
+              throw error;
+            }
+          },
+        },
+      })
+    : options.model;
+  const messages: ModelMessage[] = [
+    ...(options.messageHistory ?? []),
+    {
+      role: 'user',
+      content: [
+        { type: 'text', text: options.userMessage },
+        ...validPages.map(page => ({
+          type: 'file' as const,
+          data: page,
+          mediaType: 'image/png' as const,
+        })),
+      ],
+    },
+  ];
+  let result: Awaited<ReturnType<typeof plugins.generateText>>;
+  try {
+    result = await plugins.generateText({
+      model,
+      instructions: options.systemMessage,
+      messages,
+      providerOptions: options.providerOptions,
+      abortSignal,
+      ...(modelCalls
+        ? {
+            onLanguageModelCallEnd: ({ modelId, usage: callUsage }) => {
+              modelCalls.end(modelId, callUsage);
+            },
+          }
+        : {}),
+    });
+    modelCalls?.settle('missing');
+  } catch (error) {
+    modelCalls?.settle(abortSignal?.aborted ? 'aborted' : 'failed');
+    throw error;
+  }
   const usage = result.totalUsage;
   const inputTokens = usage.inputTokens ?? 0;
   const outputTokens = usage.outputTokens ?? 0;

package/ts_document/plugins.ts

@@ -1,3 +1,5 @@
-import { generateText } from 'ai';
+import { generateText, wrapLanguageModel } from 'ai';
 import * as smartpdf from '@push.rocks/smartpdf';
-export { generateText, smartpdf };
+import { AgentModelCallUsageRecorder } from '@modelprofile.com/flexharness-agent';
+export { generateText, wrapLanguageModel, smartpdf, AgentModelCallUsageRecorder };
+export type { TAgentModelCallUsageReporter } from '@modelprofile.com/flexharness-agent';

package/ts_document/readme.md

@@ -23,11 +23,12 @@ Supply a model from your selected FlexHarness provider. `extractTextFromPdf` rep
 
 `analyzeDocuments({ model, userMessage, pdfDocuments, systemMessage?, messageHistory? })` supports general analysis across multiple PDFs. This component owns a shared SmartPDF instance; call `stopSmartpdf()` when all document work has finished. Image-only OCR is available independently in `flexharness/ocr`.
 
-Both functions accept three request options:
+Both functions accept four request options:
 
 - `providerOptions` is passed to the model call unchanged, for example `{ openai: { store: false } }` so the provider does not keep the request as application state.
 - `abortSignal` is checked before each document is rendered and before the model call, and is passed to the model call. SmartPDF cannot stop a document it is already rendering, so that render finishes first.
 - `maxPages` is the most rendered pages one request may send to the model. A request with more pages rejects with `DocumentPageLimitError` (`renderedPages`, `maxPages`) before the model is called, and documents after the one that exceeded the limit are not rendered.
+- `onUsage` receives the usage of each model call of the request, also of a request that fails or is aborted; see [Usage of a request that fails or is aborted](#usage-of-a-request-that-fails-or-is-aborted).
 
 `extractTextFromPdfWithUsage` and `analyzeDocumentsWithUsage` take the same arguments and return `{ text, usage, pageCount }`. `usage` has the shape of an agent run's usage (`inputTokens`, `outputTokens`, `totalTokens`, `cacheReadTokens`, `cacheWriteTokens`), summed over the request's model steps, with usage the provider leaves out counted as zero; `pageCount` is the number of rendered pages sent to the model.
 
@@ -49,6 +50,28 @@ try {
 }
 ```
 
+### Usage of a request that fails or is aborted
+
+A request that throws returns no `usage`. `onUsage` reports each model call of a request once, as soon as its usage is known, whatever the outcome, with the events of `AgentModelCallUsageRecorder` (the type is `TFlexModelCallUsage` from `@modelprofile.com/flexharness`):
+
+```typescript
+import type { TFlexModelCallUsage } from '@modelprofile.com/flexharness';
+
+const calls: TFlexModelCallUsage[] = [];
+try {
+  await extractTextFromPdfWithUsage(pdfBytes, { model, abortSignal, onUsage: (call) => calls.push(call) });
+} finally {
+  const reported = calls.filter((call) => call.status === 'reported');
+  const complete = reported.length === calls.length; // an unreported call used an unknown number of tokens
+}
+```
+
+- `status: 'reported'`: the provider reported the call's `usage`; `requestedModelId` is the model's id and `responseModelId` the id the response named.
+- `status: 'unreported'`: the call `failed`, was `aborted`, or its response carried no usage (`missing`). The provider may still have consumed tokens for it.
+- Every attempt is a call of its own: a call retried after a failure reports the failed attempt and the retry.
+- A request that ends before its model is called (an abort during rendering, `DocumentPageLimitError`) reports nothing.
+- When a request returns, its reported calls sum to its `usage`; count one or the other. `onUsage` must not throw.
+
 ## Issue Reporting and Security
 
 Report issues and security concerns through [community.foss.global](https://community.foss.global).