@modelprofile.com/flexharness-agent 8.3.0 → 8.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts_agent/classes.sessionengine.d.ts +3 -0
- package/dist_ts_agent/classes.sessionengine.js +43 -11
- package/dist_ts_agent/index.d.ts +3 -1
- package/dist_ts_agent/index.js +2 -1
- package/dist_ts_agent/plugins.d.ts +1 -1
- package/dist_ts_agent/plugins.js +1 -1
- package/dist_ts_agent/runner.d.ts +1 -1
- package/dist_ts_agent/runtime.run.js +2 -1
- package/dist_ts_agent/smartagent.interfaces.d.ts +77 -8
- package/dist_ts_agent/smartagent.interfaces.js +1 -1
- package/dist_ts_agent/smartagent.usage.d.ts +25 -0
- package/dist_ts_agent/smartagent.usage.js +53 -0
- package/package.json +2 -2
- package/readme.md +62 -3
- package/ts_agent/classes.sessionengine.ts +61 -8
- package/ts_agent/index.ts +7 -0
- package/ts_agent/plugins.ts +1 -0
- package/ts_agent/readme.md +62 -3
- package/ts_agent/runner.ts +8 -1
- package/ts_agent/runtime.run.ts +1 -0
- package/ts_agent/smartagent.interfaces.ts +82 -8
- package/ts_agent/smartagent.usage.ts +66 -0
package/dist_ts_agent/index.d.ts
CHANGED
|
@@ -4,9 +4,11 @@ export { buildModelMessages } from './smartagent.context.js';
|
|
|
4
4
|
export { filterModelVisibleAgentEvents, getAgentGenerationTransactions, getOpenAgentGenerationTransactions, getUncertainToolExecutionIntents, isTransactionalGeneration, } from './smartagent.transactions.js';
|
|
5
5
|
export { createAgentEvent, createAgentEventId, modelMessagesToAgentEvents, } from './smartagent.events.js';
|
|
6
6
|
export { ToolRegistry } from './smartagent.classes.toolregistry.js';
|
|
7
|
+
export { AgentModelCallUsageRecorder } from './smartagent.usage.js';
|
|
8
|
+
export type { IAgentModelCallUsageRecorderModel } from './smartagent.usage.js';
|
|
7
9
|
export { AgentGenerationLeaseCleanupError, ContextOverflowError, } from './smartagent.interfaces.js';
|
|
8
10
|
export { AgentEventStoreConflictError, InMemoryAgentEventStore, isAgentEventStoreV2, validateAgentEventArchiveV2, validateAgentEventSnapshotV2, } from './smartagent.persistence.js';
|
|
9
|
-
export type { IAgentCacheOptions, IAgentContextOverflowInvocationOptions, IAgentContextBuildOptions, IAgentContextCompactionOptions, IAgentCompactOptions, IAgentEventRetentionOptions, IAgentBeginGenerationOptions, IAgentGenerateOptions, IAgentGenerateResult, IAgentGenerationHandle, IAgentGenerationLease, IAgentGenerationPreparationContext, IAgentRunOptions, IAgentRunResult, IAgentSession, IAgentSessionAbortOptions, IAgentSessionChange, IAgentSessionOptions, IAgentScheduleGenerateOptions, IAgentToolCallRecord, IAgentToolCallStartEvent, IAgentToolCallUpdateEvent, ProviderOptions, TAgentCacheRetention, TAgentCacheSetting, TAgentPrompt, TAgentContextBuilder, TAgentContextCompactor, TAgentGenerationPrepare, TAgentSessionChangeListener, TAgentToolCallFinishEvent, TAgentToolExecutionReconciliationOptions, } from './smartagent.interfaces.js';
|
|
11
|
+
export type { IAgentCacheOptions, IAgentContextOverflowInvocationOptions, IAgentContextBuildOptions, IAgentContextCompactionOptions, IAgentCompactOptions, IAgentEventRetentionOptions, IAgentBeginGenerationOptions, IAgentGenerateOptions, IAgentGenerateResult, IAgentGenerationHandle, IAgentGenerationLease, IAgentGenerationPreparationContext, IAgentRunOptions, IAgentRunResult, IAgentSession, IAgentSessionAbortOptions, IAgentSessionChange, IAgentSessionOptions, IAgentScheduleGenerateOptions, IAgentToolCallRecord, IAgentToolCallStartEvent, IAgentToolCallUpdateEvent, IAgentUsage, ProviderOptions, TAgentCacheRetention, TAgentCacheSetting, TAgentPrompt, TAgentContextBuilder, TAgentContextCompactor, TAgentGenerationPrepare, TAgentModelCallUnreportedReason, TAgentModelCallUsage, TAgentModelCallUsageEvent, TAgentModelCallUsageReporter, TAgentSessionChangeListener, TAgentToolCallFinishEvent, TAgentToolExecutionReconciliationOptions, } from './smartagent.interfaces.js';
|
|
10
12
|
export type { IAgentEventArchive, IAgentEventArchiveV2, IAgentEventSnapshot, IAgentEventSnapshotV2, IAgentEventStore, IAgentEventStoreV2, TAgentEventArchive, TAgentEventSnapshot, TAgentEventStore, } from './smartagent.persistence.js';
|
|
11
13
|
export type { IAgentEventBase, IAgentRuntimeEventPayload, IArchivedAgentGenerationTransaction, IAssistantMessageEvent, IContextCompactionEvent, IGenerationBegunEvent, IGenerationExecutionCompletedEvent, IGenerationExecutionStartedEvent, IGenerationOutcomeEvent, IModelMessageEvent, IModelMessagesToEventsOptions, IModelMessageEventIdentityCoordinates, IRuntimeEvent, ISystemMessageEvent, IToolCallEvent, IToolExecutionIntentEvent, IToolExecutionReconciliationEvent, IToolResultEvent, IUserMessageEvent, TAgentAssistantMessage, TAgentConversationEvent, TAgentControlEvent, TAgentEvent, TAgentEventInput, TAgentSystemMessage, TAgentGenerationOutcome, TAgentToolCallPart, TAgentToolMessage, TAgentToolResultOutput, TAgentToolResultPart, TAgentUserMessage, TModelMessageEmittedEventKind, TModelMessageEventIdentityFactory, TToolExecutionReconciliation, } from './smartagent.events.js';
|
|
12
14
|
export type { IAgentGenerationTransaction, TAgentGenerationTransactionState, } from './smartagent.transactions.js';
|
package/dist_ts_agent/index.js
CHANGED
|
@@ -4,6 +4,7 @@ export { buildModelMessages } from './smartagent.context.js';
|
|
|
4
4
|
export { filterModelVisibleAgentEvents, getAgentGenerationTransactions, getOpenAgentGenerationTransactions, getUncertainToolExecutionIntents, isTransactionalGeneration, } from './smartagent.transactions.js';
|
|
5
5
|
export { createAgentEvent, createAgentEventId, modelMessagesToAgentEvents, } from './smartagent.events.js';
|
|
6
6
|
export { ToolRegistry } from './smartagent.classes.toolregistry.js';
|
|
7
|
+
export { AgentModelCallUsageRecorder } from './smartagent.usage.js';
|
|
7
8
|
export { AgentGenerationLeaseCleanupError, ContextOverflowError, } from './smartagent.interfaces.js';
|
|
8
9
|
export { AgentEventStoreConflictError, InMemoryAgentEventStore, isAgentEventStoreV2, validateAgentEventArchiveV2, validateAgentEventSnapshotV2, } from './smartagent.persistence.js';
|
|
9
10
|
export * from './tool.contracts.js';
|
|
@@ -12,4 +13,4 @@ export * from './tool.adapter.js';
|
|
|
12
13
|
export { tool, jsonSchema, stepCountIs } from 'ai';
|
|
13
14
|
export { z } from 'zod';
|
|
14
15
|
export { FileAgentEventStore } from './persistence.file.js';
|
|
15
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
16
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90c19hZ2VudC9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsUUFBUSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFDekQsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBQy9ELE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzdELE9BQU8sRUFDTCw2QkFBNkIsRUFDN0IsOEJBQThCLEVBQzlCLGtDQUFrQyxFQUNsQyxnQ0FBZ0MsRUFDaEMseUJBQXlCLEdBQzFCLE1BQU0sOEJBQThCLENBQUM7QUFDdEMsT0FBTyxFQUNMLGdCQUFnQixFQUNoQixrQkFBa0IsRUFDbEIsMEJBQTBCLEdBQzNCLE1BQU0sd0JBQXdCLENBQUM7QUFDaEMsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLHNDQUFzQyxDQUFDO0FBQ3BFLE9BQU8sRUFBRSwyQkFBMkIsRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBRXBFLE9BQU8sRUFDTCxnQ0FBZ0MsRUFDaEMsb0JBQW9CLEdBQ3JCLE1BQU0sNEJBQTRCLENBQUM7QUFDcEMsT0FBTyxFQUNMLDRCQUE0QixFQUM1Qix1QkFBdUIsRUFDdkIsbUJBQW1CLEVBQ25CLDJCQUEyQixFQUMzQiw0QkFBNEIsR0FDN0IsTUFBTSw2QkFBNkIsQ0FBQztBQTRGckMsY0FBYyxxQkFBcUIsQ0FBQztBQUNwQyxjQUFjLHVCQUF1QixDQUFDO0FBQ3RDLGNBQWMsbUJBQW1CLENBQUM7QUFDbEMsT0FBTyxFQUFFLElBQUksRUFBRSxVQUFVLEVBQUUsV0FBVyxFQUFFLE1BQU0sSUFBSSxDQUFDO0FBQ25ELE9BQU8sRUFBRSxDQUFDLEVBQUUsTUFBTSxLQUFLLENBQUM7QUFFeEIsT0FBTyxFQUFFLG1CQUFtQixFQUFvQyxNQUFNLHVCQUF1QixDQUFDIn0=
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { streamText, generateText, stepCountIs, wrapLanguageModel } from 'ai';
|
|
2
2
|
export { streamText, generateText, stepCountIs, wrapLanguageModel };
|
|
3
|
-
export type { AssistantModelMessage, ModelMessage, StepResult, SystemModelMessage, ToolSet, ToolModelMessage, ToolExecutionOptions, StreamTextResult, UserModelMessage, } from 'ai';
|
|
3
|
+
export type { AssistantModelMessage, LanguageModelUsage, ModelMessage, StepResult, SystemModelMessage, ToolSet, ToolModelMessage, ToolExecutionOptions, StreamTextResult, UserModelMessage, } from 'ai';
|
|
4
4
|
import { applySmartAiCacheProviderOptions, createModelLimitInfo, createSmartAiCachingMiddleware, isModelLimitError, isModelLimitInfo, jsonSchema, ModelLimitError, readRetryAfterMs, resolveSmartAiCacheProvider, tool } from '@modelprofile.com/flexharness-models';
|
|
5
5
|
export { applySmartAiCacheProviderOptions, createModelLimitInfo, createSmartAiCachingMiddleware, isModelLimitError, isModelLimitInfo, ModelLimitError, readRetryAfterMs, resolveSmartAiCacheProvider, tool, jsonSchema, };
|
|
6
6
|
export type { ISmartAiCacheOptions, TSmartAiLanguageModel, TSmartAiCacheRetention, TSmartAiCacheSetting, TSmartAiProviderOptions as ProviderOptions, } from '@modelprofile.com/flexharness-models';
|
package/dist_ts_agent/plugins.js
CHANGED
|
@@ -7,4 +7,4 @@ export { applySmartAiCacheProviderOptions, createModelLimitInfo, createSmartAiCa
|
|
|
7
7
|
// zod
|
|
8
8
|
import { z } from 'zod';
|
|
9
9
|
export { z };
|
|
10
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
10
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGx1Z2lucy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3RzX2FnZW50L3BsdWdpbnMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FBYztBQUNkLE9BQU8sRUFBRSxVQUFVLEVBQUUsWUFBWSxFQUFFLFdBQVcsRUFBRSxpQkFBaUIsRUFBRSxNQUFNLElBQUksQ0FBQztBQUU5RSxPQUFPLEVBQUUsVUFBVSxFQUFFLFlBQVksRUFBRSxXQUFXLEVBQUUsaUJBQWlCLEVBQUUsQ0FBQztBQWVwRSw2QkFBNkI7QUFDN0IsT0FBTyxFQUNMLGdDQUFnQyxFQUNoQyxvQkFBb0IsRUFDcEIsOEJBQThCLEVBQzlCLGlCQUFpQixFQUNqQixnQkFBZ0IsRUFDaEIsVUFBVSxFQUNWLGVBQWUsRUFDZixnQkFBZ0IsRUFDaEIsMkJBQTJCLEVBQzNCLElBQUksR0FDTCxNQUFNLHNDQUFzQyxDQUFDO0FBRTlDLE9BQU8sRUFDTCxnQ0FBZ0MsRUFDaEMsb0JBQW9CLEVBQ3BCLDhCQUE4QixFQUM5QixpQkFBaUIsRUFDakIsZ0JBQWdCLEVBQ2hCLGVBQWUsRUFDZixnQkFBZ0IsRUFDaEIsMkJBQTJCLEVBQzNCLElBQUksRUFDSixVQUFVLEdBQ1gsQ0FBQztBQWdCRixNQUFNO0FBQ04sT0FBTyxFQUFFLENBQUMsRUFBRSxNQUFNLEtBQUssQ0FBQztBQUV4QixPQUFPLEVBQUUsQ0FBQyxFQUFFLENBQUMifQ==
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import type { IAgentRunOptions, IAgentRunResult } from './smartagent.interfaces.js';
|
|
2
2
|
/** Run in Node or a secure browser context, with internally owned cleanup. */
|
|
3
3
|
export declare function runAgent(options: IAgentRunOptions): Promise<IAgentRunResult>;
|
|
4
|
-
export type { IAgentRunOptions, IAgentRunResult } from './smartagent.interfaces.js';
|
|
4
|
+
export type { IAgentRunOptions, IAgentRunResult, IAgentUsage, TAgentModelCallUnreportedReason, TAgentModelCallUsage, TAgentModelCallUsageEvent, } from './smartagent.interfaces.js';
|
|
@@ -39,6 +39,7 @@ export async function runAgentWithSession(options, createSession) {
|
|
|
39
39
|
onToolCallUpdate: options.onToolCallUpdate,
|
|
40
40
|
onToolCallFinish: options.onToolCallFinish,
|
|
41
41
|
onRetry: options.onRetry,
|
|
42
|
+
onUsage: options.onUsage,
|
|
42
43
|
onToolCall: options.onToolCall,
|
|
43
44
|
onToolResult: options.onToolResult,
|
|
44
45
|
onContextOverflow: options.onContextOverflow,
|
|
@@ -92,4 +93,4 @@ export async function runAgentWithSession(options, createSession) {
|
|
|
92
93
|
await session.close();
|
|
93
94
|
}
|
|
94
95
|
}
|
|
95
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
96
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVudGltZS5ydW4uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90c19hZ2VudC9ydW50aW1lLnJ1bi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFPQSxNQUFNLGNBQWMsR0FBRyxDQUNyQixNQUE4QixFQUM5QixNQUF1QyxFQUNqQyxFQUFFO0lBQ1IsTUFBTSxPQUFPLEdBQUcsSUFBSSxHQUFHLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFDLE1BQU0sRUFBRSxLQUFLLEVBQUUsRUFBRSxDQUFDLENBQUMsTUFBTSxDQUFDLFVBQVUsRUFBRSxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDbkYsS0FBSyxNQUFNLE1BQU0sSUFBSSxNQUFNLEVBQUUsQ0FBQztRQUM1QixNQUFNLGFBQWEsR0FBRyxPQUFPLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxVQUFVLENBQUMsQ0FBQztRQUNyRCxJQUFJLGFBQWEsS0FBSyxTQUFTLEVBQUUsQ0FBQztZQUNoQyxNQUFNLENBQUMsSUFBSSxDQUFDLEVBQUUsR0FBRyxNQUFNLEVBQUUsQ0FBQyxDQUFDO1lBQzNCLE9BQU8sQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLFVBQVUsRUFBRSxNQUFNLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDO1FBQ3BELENBQUM7YUFBTSxDQUFDO1lBQ04sTUFBTSxDQUFDLGFBQWEsQ0FBQyxHQUFHLEVBQUUsR0FBRyxNQUFNLENBQUMsYUFBYSxDQUFDLEVBQUUsR0FBRyxNQUFNLEVBQUUsQ0FBQztRQUNsRSxDQUFDO0lBQ0gsQ0FBQztBQUNILENBQUMsQ0FBQztBQUVGLE1BQU0sQ0FBQyxLQUFLLFVBQVUsbUJBQW1CLENBQ3ZDLE9BQXlCLEVBQ3pCLGFBQXdFO0lBRXhFLE1BQU0sT0FBTyxHQUFHLE1BQU0sYUFBYSxDQUFDO1FBQ2xDLEtBQUssRUFBRSxPQUFPLENBQUMsS0FBSztRQUNwQixNQUFNLEVBQUUsT0FBTyxDQUFDLE1BQU07UUFDdEIsS0FBSyxFQUFFLE9BQU8sQ0FBQyxLQUFLO1FBQ3BCLGVBQWUsRUFBRSxPQUFPLENBQUMsZUFBZTtRQUN4QyxTQUFTLEVBQUUsT0FBTyxDQUFDLFNBQVM7UUFDNUIsVUFBVSxFQUFFLE9BQU8sQ0FBQyxVQUFVO1FBQzlCLEtBQUssRUFBRSxPQUFPLENBQUMsS0FBSztRQUNwQixRQUFRLEVBQUUsT0FBTyxDQUFDLFFBQVE7UUFDMUIsUUFBUSxFQUFFLE9BQU8sQ0FBQyxRQUFRO1FBQzFCLE1BQU0sRUFBRSxPQUFPLENBQUMsTUFBTTtRQUN0QixnQkFBZ0IsRUFBRSxPQUFPLENBQUMsZ0JBQWdCO1FBQzFDLGNBQWMsRUFBRSxPQUFPLENBQUMsY0FBYztRQUN0QyxnQkFBZ0IsRUFBRSxPQUFPLENBQUMsZ0JBQWdCO1FBQzFDLGNBQWMsRUFBRSxPQUFPLENBQUMsY0FBYztRQUN0Qyx1QkFBdUIsRUFBRSxPQUFPLENBQUMsdUJBQXVCO1FBQ3hELHdCQUF3QixFQUFFLE9BQU8sQ0FBQyx3QkFBd0I7UUFDMUQsK0JBQStCLEVBQUUsT0FBTyxDQUFDLCtCQUErQjtRQUN4RSxnQ0FBZ0MsRUFBRSxPQUFPLENBQUMsZ0NBQWdDO1FBQzFFLE9BQU8sRUFBRSxPQUFPLENBQUMsT0FBTztRQUN4QixnQkFBZ0IsRUFBRSxPQUFPLENBQUMsZ0JBQWdCO1FBQzFDLGdCQUFnQixFQUFFLE9BQU8sQ0FBQyxnQkFBZ0I7UUFDMUMsY0FBYyxFQUFFLE9BQU8sQ0FBQyxjQUFjO1FBQ3RDLGVBQWUsRUFBRSxPQUFPLENBQUMsZUFBZTtRQUN4QyxnQkFBZ0IsRUFBRSxPQUFPLENBQUMsZ0JBQWdCO1FBQzFDLGdCQUFnQixFQUFFLE9BQU8sQ0FBQyxnQkFBZ0I7UUFDMUMsT0FBTyxFQUFFLE9BQU8sQ0FBQyxPQUFPO1FBQ3hCLE9BQU8sRUFBRSxPQUFPLENBQUMsT0FBTztRQUN4QixVQUFVLEVBQUUsT0FBTyxDQUFDLFVBQVU7UUFDOUIsWUFBWSxFQUFFLE9BQU8sQ0FBQyxZQUFZO1FBQ2xDLGlCQUFpQixFQUFFLE9BQU8sQ0FBQyxpQkFBaUI7UUFDNUMseUJBQXlCLEVBQUUsT0FBTyxDQUFDLHlCQUF5QjtLQUM3RCxDQUFDLENBQUM7SUFDSCxJQUFJLFVBQVUsR0FBRyxDQUFDLENBQUM7SUFDbkIsSUFBSSxVQUFVLEdBQUcsQ0FBQyxDQUFDO0lBQ25CLElBQUksV0FBVyxHQUFHLENBQUMsQ0FBQztJQUNwQixJQUFJLGNBQWMsR0FBRyxDQUFDLENBQUM7SUFDdkIsSUFBSSxlQUFlLEdBQUcsQ0FBQyxDQUFDO0lBQ3hCLElBQUksaUJBQWlCLEdBQUcsQ0FBQyxDQUFDO0lBQzFCLE1BQU0sU0FBUyxHQUEyQixFQUFFLENBQUM7SUFFN0MsSUFBSSxDQUFDO1FBQ0gsTUFBTSxPQUFPLENBQUMsZUFBZSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQztRQUM5QyxPQUFPLElBQUksRUFBRSxDQUFDO1lBQ1osTUFBTSxnQkFBZ0IsR0FBRyxNQUFNLE9BQU8sQ0FBQyxRQUFRLENBQUM7Z0JBQzlDLFFBQVEsRUFBRSxPQUFPLENBQUMsUUFBUTtnQkFDMUIsS0FBSyxFQUFFLE9BQU8sQ0FBQyxLQUFLO2FBQ3JCLENBQUMsQ0FBQztZQUNILFVBQVUsSUFBSSxnQkFBZ0IsQ0FBQyxLQUFLLENBQUM7WUFDckMsVUFBVSxJQUFJLGdCQUFnQixDQUFDLEtBQUssQ0FBQyxXQUFXLENBQUM7WUFDakQsV0FBVyxJQUFJLGdCQUFnQixDQUFDLEtBQUssQ0FBQyxZQUFZLENBQUM7WUFDbkQsY0FBYyxJQUFJLGdCQUFnQixDQUFDLEtBQUssQ0FBQyxlQUFlLENBQUM7WUFDekQsZUFBZSxJQUFJLGdCQUFnQixDQUFDLEtBQUssQ0FBQyxnQkFBZ0IsQ0FBQztZQUMzRCxjQUFjLENBQUMsU0FBUyxFQUFFLGdCQUFnQixDQUFDLFNBQVMsQ0FBQyxDQUFDO1lBRXRELE1BQU0sU0FBUyxHQUFvQjtnQkFDakMsSUFBSSxFQUFFLGdCQUFnQixDQUFDLElBQUk7Z0JBQzNCLFFBQVEsRUFBRSxPQUFPLENBQUMsZ0JBQWdCLEVBQUU7Z0JBQ3BDLEtBQUssRUFBRSxVQUFVO2dCQUNqQixZQUFZLEVBQUUsZ0JBQWdCLENBQUMsWUFBWTtnQkFDM0MsS0FBSyxFQUFFO29CQUNMLFdBQVcsRUFBRSxVQUFVO29CQUN2QixZQUFZLEVBQUUsV0FBVztvQkFDekIsV0FBVyxFQUFFLFVBQVUsR0FBRyxXQUFXO29CQUNyQyxlQUFlLEVBQUUsY0FBYztvQkFDL0IsZ0JBQWdCLEVBQUUsZUFBZTtpQkFDbEM7Z0JBQ0QsU0FBUzthQUNWLENBQUM7WUFFRixNQUFNLGdCQUFnQixHQUFHLE1BQU0sT0FBTyxDQUFDLGtCQUFrQixFQUFFLENBQUMsU0FBUyxDQUFDLENBQUM7WUFDdkUsSUFBSSxPQUFPLGdCQUFnQixLQUFLLFFBQVE7Z0JBQUUsT0FBTyxTQUFTLENBQUM7WUFDM0QsSUFBSSxpQkFBaUIsSUFBSSxDQUFDLE9BQU8sQ0FBQyxvQkFBb0IsSUFBSSxDQUFDLENBQUMsRUFBRSxDQUFDO2dCQUM3RCxNQUFNLElBQUksS0FBSyxDQUFDLHVDQUF1QyxnQkFBZ0IsRUFBRSxDQUFDLENBQUM7WUFDN0UsQ0FBQztZQUNELGlCQUFpQixFQUFFLENBQUM7WUFDcEIsTUFBTSxPQUFPLENBQUMsZUFBZSxDQUFDLGdCQUFnQixDQUFDLENBQUM7UUFDbEQsQ0FBQztJQUNILENBQUM7WUFBUyxDQUFDO1FBQ1QsTUFBTSxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDeEIsQ0FBQztBQUNILENBQUMifQ==
|
|
@@ -33,8 +33,64 @@ export type TAgentToolCallFinishEvent = (IAgentToolCallStartEvent & {
|
|
|
33
33
|
success: false;
|
|
34
34
|
error: string;
|
|
35
35
|
});
|
|
36
|
+
/** Token usage of model calls. A count a provider leaves out counts as zero. */
|
|
37
|
+
export interface IAgentUsage {
|
|
38
|
+
inputTokens: number;
|
|
39
|
+
outputTokens: number;
|
|
40
|
+
totalTokens: number;
|
|
41
|
+
cacheReadTokens: number;
|
|
42
|
+
cacheWriteTokens: number;
|
|
43
|
+
}
|
|
44
|
+
interface IAgentModelCallIdentity {
|
|
45
|
+
/** The provider of the model that was called, as the language model names it. */
|
|
46
|
+
provider: string;
|
|
47
|
+
/**
|
|
48
|
+
* The model id the call was made with: the language model's `modelId`. Every call carries it,
|
|
49
|
+
* whatever its outcome, so key usage caps on `provider` and `requestedModelId`.
|
|
50
|
+
*/
|
|
51
|
+
requestedModelId: string;
|
|
52
|
+
}
|
|
53
|
+
/** Why a model call ended without the provider reporting its usage. */
|
|
54
|
+
export type TAgentModelCallUnreportedReason = 'aborted' | 'failed' | 'missing';
|
|
55
|
+
/**
|
|
56
|
+
* The usage outcome of one model call, as the component that made the call reports it.
|
|
57
|
+
* `reported`: the provider reported the call's usage. `responseModelId` is the model id the
|
|
58
|
+
* provider's response named, or `requestedModelId` when the response named none.
|
|
59
|
+
* `unreported`: the call ended before the provider reported its usage: `aborted` (the call was
|
|
60
|
+
* aborted), `failed` (the call or its response failed) or `missing` (the response carried no usage).
|
|
61
|
+
* The provider may still have consumed tokens for it; their number is unknown.
|
|
62
|
+
*/
|
|
63
|
+
export type TAgentModelCallUsage = (IAgentModelCallIdentity & {
|
|
64
|
+
status: 'reported';
|
|
65
|
+
responseModelId: string;
|
|
66
|
+
usage: IAgentUsage;
|
|
67
|
+
}) | (IAgentModelCallIdentity & {
|
|
68
|
+
status: 'unreported';
|
|
69
|
+
reason: TAgentModelCallUnreportedReason;
|
|
70
|
+
});
|
|
71
|
+
/** Receives the usage outcome of each model call. Must not throw. */
|
|
72
|
+
export type TAgentModelCallUsageReporter = (call: TAgentModelCallUsage) => void;
|
|
73
|
+
/**
|
|
74
|
+
* The usage outcome of one model call of a session, reported once per call through `onUsage`.
|
|
75
|
+
* `source: 'generation'`: a model step of the generation `generationId`.
|
|
76
|
+
* `source: 'compaction'`: a model call of a context compaction. `generationId` names the generation
|
|
77
|
+
* whose context overflow caused it; it is absent for `compact()` and event retention.
|
|
78
|
+
*/
|
|
79
|
+
export type TAgentModelCallUsageEvent = TAgentModelCallUsage & ({
|
|
80
|
+
source: 'generation';
|
|
81
|
+
generationId: string;
|
|
82
|
+
} | {
|
|
83
|
+
source: 'compaction';
|
|
84
|
+
generationId?: string;
|
|
85
|
+
});
|
|
36
86
|
export interface IAgentContextOverflowInvocationOptions {
|
|
37
87
|
abortSignal?: AbortSignal;
|
|
88
|
+
/**
|
|
89
|
+
* Reports the usage of each model call the handler makes, into the session's `onUsage` as
|
|
90
|
+
* `source: 'compaction'`. The session always supplies it. Report every call before the returned
|
|
91
|
+
* promise settles, including calls that fail or are aborted.
|
|
92
|
+
*/
|
|
93
|
+
reportUsage?: TAgentModelCallUsageReporter;
|
|
38
94
|
}
|
|
39
95
|
export interface IAgentContextBuildOptions {
|
|
40
96
|
events: readonly TAgentEvent[];
|
|
@@ -43,6 +99,12 @@ export type TAgentContextBuilder = (options: IAgentContextBuildOptions) => Model
|
|
|
43
99
|
export interface IAgentContextCompactionOptions {
|
|
44
100
|
abortSignal?: AbortSignal;
|
|
45
101
|
reason: 'context-overflow' | 'retention' | 'manual';
|
|
102
|
+
/**
|
|
103
|
+
* Reports the usage of each model call the compactor makes, into the session's `onUsage` as
|
|
104
|
+
* `source: 'compaction'`. The session always supplies it. Report every call before the returned
|
|
105
|
+
* promise settles, including calls that fail or are aborted.
|
|
106
|
+
*/
|
|
107
|
+
reportUsage?: TAgentModelCallUsageReporter;
|
|
46
108
|
}
|
|
47
109
|
export type TAgentContextCompactor = (messages: ModelMessage[], events: readonly TAgentEvent[], options: IAgentContextCompactionOptions) => Promise<ModelMessage[]>;
|
|
48
110
|
export interface IAgentEventRetentionOptions {
|
|
@@ -116,6 +178,16 @@ export interface IAgentSessionOptions {
|
|
|
116
178
|
onToolCallFinish?: (event: TAgentToolCallFinishEvent) => void;
|
|
117
179
|
/** Called before the engine waits to retry a rate-limited or unavailable model call. */
|
|
118
180
|
onRetry?: (event: IAgentRetryEvent) => void;
|
|
181
|
+
/**
|
|
182
|
+
* Called once for every model call of the session, as soon as its usage is known: when the
|
|
183
|
+
* provider reports it, or when the call ends without it. This covers the generation's model
|
|
184
|
+
* steps, the model call of `compactMessages()` when a compactor passes it `reportUsage`, and every
|
|
185
|
+
* call a `contextCompactor` or `onContextOverflow` handler reports through `reportUsage`. The
|
|
186
|
+
* reported calls of a generation, including those of a compaction its context overflow caused,
|
|
187
|
+
* sum to the generation's `usage` when it returns. Must not throw; an error it throws is reported
|
|
188
|
+
* like a session listener error and does not change the generation's outcome.
|
|
189
|
+
*/
|
|
190
|
+
onUsage?: (event: TAgentModelCallUsageEvent) => void;
|
|
119
191
|
/** @deprecated Use onToolCallStart instead. */
|
|
120
192
|
onToolCall?: (toolName: string, input: unknown) => void;
|
|
121
193
|
/** @deprecated Use onToolCallFinish instead. */
|
|
@@ -229,14 +301,11 @@ export interface IAgentRunResult {
|
|
|
229
301
|
steps: number;
|
|
230
302
|
/** Finish reason from the final step */
|
|
231
303
|
finishReason: string;
|
|
232
|
-
/**
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
cacheReadTokens: number;
|
|
238
|
-
cacheWriteTokens: number;
|
|
239
|
-
};
|
|
304
|
+
/**
|
|
305
|
+
* Token usage the provider reported for every model call of the run: its model steps, retried
|
|
306
|
+
* calls included, and the calls of a compaction its context overflow caused that were reported
|
|
307
|
+
*/
|
|
308
|
+
usage: IAgentUsage;
|
|
240
309
|
/** Tool calls observed during the run, including inputs and outputs/errors when available */
|
|
241
310
|
toolCalls: IAgentToolCallRecord[];
|
|
242
311
|
}
|
|
@@ -21,4 +21,4 @@ export class ContextOverflowError extends Error {
|
|
|
21
21
|
this.name = 'ContextOverflowError';
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
24
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRhZ2VudC5pbnRlcmZhY2VzLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHNfYWdlbnQvc21hcnRhZ2VudC5pbnRlcmZhY2VzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQXNTQSxNQUFNLE9BQU8sZ0NBQWlDLFNBQVEsS0FBSztJQUN6QyxLQUFLLENBQXdCO0lBQzdCLFlBQVksQ0FBVTtJQUNyQixvQkFBb0IsQ0FBc0I7SUFFM0QsWUFDRSxLQUE0QixFQUM1QixZQUFxQixFQUNyQixvQkFBeUM7UUFFekMsS0FBSyxDQUNILG9DQUFvQyxZQUFZLFlBQVksS0FBSztZQUMvRCxDQUFDLENBQUMsWUFBWSxDQUFDLE9BQU87WUFDdEIsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUMzQixDQUFDO1FBQ0YsSUFBSSxDQUFDLElBQUksR0FBRyxrQ0FBa0MsQ0FBQztRQUMvQyxJQUFJLENBQUMsS0FBSyxHQUFHLEtBQUssQ0FBQztRQUNuQixJQUFJLENBQUMsWUFBWSxHQUFHLFlBQVksQ0FBQztRQUNqQyxJQUFJLENBQUMsb0JBQW9CLEdBQUcsb0JBQW9CLENBQUM7SUFDbkQsQ0FBQztJQUVNLFlBQVk7UUFDakIsT0FBTyxJQUFJLENBQUMsb0JBQW9CLEVBQUUsQ0FBQztJQUNyQyxDQUFDO0NBQ0Y7QUFzSEQsTUFBTSxPQUFPLG9CQUFxQixTQUFRLEtBQUs7SUFDN0MsWUFBWSxPQUFPLEdBQUcsdUVBQXVFO1FBQzNGLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQztRQUNmLElBQUksQ0FBQyxJQUFJLEdBQUcsc0JBQXNCLENBQUM7SUFDckMsQ0FBQztDQUNGIn0=
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { LanguageModelUsage } from './plugins.js';
|
|
2
|
+
import type { TAgentModelCallUnreportedReason, TAgentModelCallUsageReporter } from './smartagent.interfaces.js';
|
|
3
|
+
/** The identity of the language model whose calls a recorder follows. */
|
|
4
|
+
export interface IAgentModelCallUsageRecorderModel {
|
|
5
|
+
readonly provider: string;
|
|
6
|
+
readonly modelId: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Follows the calls of one language model and reports each call's usage exactly once.
|
|
10
|
+
* Call `start()` when a model call begins, `end()` when its provider reports usage, and `settle()`
|
|
11
|
+
* when the call ends otherwise. A call that is still open when the next one starts ended without
|
|
12
|
+
* usage and is reported as `unreported` with reason `missing`.
|
|
13
|
+
*/
|
|
14
|
+
export declare class AgentModelCallUsageRecorder {
|
|
15
|
+
private readonly model;
|
|
16
|
+
private readonly report;
|
|
17
|
+
private open;
|
|
18
|
+
constructor(model: IAgentModelCallUsageRecorderModel, report: TAgentModelCallUsageReporter);
|
|
19
|
+
/** A model call begins. */
|
|
20
|
+
start(): void;
|
|
21
|
+
/** The provider reported the usage of the current call. A count it leaves out counts as zero. */
|
|
22
|
+
end(responseModelId: string, providerUsage: LanguageModelUsage): void;
|
|
23
|
+
/** The current call ended without usage. Does nothing when no call is open. */
|
|
24
|
+
settle(reason: TAgentModelCallUnreportedReason): void;
|
|
25
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Follows the calls of one language model and reports each call's usage exactly once.
|
|
3
|
+
* Call `start()` when a model call begins, `end()` when its provider reports usage, and `settle()`
|
|
4
|
+
* when the call ends otherwise. A call that is still open when the next one starts ended without
|
|
5
|
+
* usage and is reported as `unreported` with reason `missing`.
|
|
6
|
+
*/
|
|
7
|
+
export class AgentModelCallUsageRecorder {
|
|
8
|
+
model;
|
|
9
|
+
report;
|
|
10
|
+
open = false;
|
|
11
|
+
constructor(model, report) {
|
|
12
|
+
this.model = model;
|
|
13
|
+
this.report = report;
|
|
14
|
+
}
|
|
15
|
+
/** A model call begins. */
|
|
16
|
+
start() {
|
|
17
|
+
this.settle('missing');
|
|
18
|
+
this.open = true;
|
|
19
|
+
}
|
|
20
|
+
/** The provider reported the usage of the current call. A count it leaves out counts as zero. */
|
|
21
|
+
end(responseModelId, providerUsage) {
|
|
22
|
+
this.open = false;
|
|
23
|
+
const inputTokens = providerUsage.inputTokens ?? 0;
|
|
24
|
+
const outputTokens = providerUsage.outputTokens ?? 0;
|
|
25
|
+
const usage = {
|
|
26
|
+
inputTokens,
|
|
27
|
+
outputTokens,
|
|
28
|
+
totalTokens: inputTokens + outputTokens,
|
|
29
|
+
cacheReadTokens: providerUsage.inputTokenDetails.cacheReadTokens ?? 0,
|
|
30
|
+
cacheWriteTokens: providerUsage.inputTokenDetails.cacheWriteTokens ?? 0,
|
|
31
|
+
};
|
|
32
|
+
this.report({
|
|
33
|
+
status: 'reported',
|
|
34
|
+
provider: this.model.provider,
|
|
35
|
+
requestedModelId: this.model.modelId,
|
|
36
|
+
responseModelId,
|
|
37
|
+
usage,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
/** The current call ended without usage. Does nothing when no call is open. */
|
|
41
|
+
settle(reason) {
|
|
42
|
+
if (!this.open)
|
|
43
|
+
return;
|
|
44
|
+
this.open = false;
|
|
45
|
+
this.report({
|
|
46
|
+
status: 'unreported',
|
|
47
|
+
provider: this.model.provider,
|
|
48
|
+
requestedModelId: this.model.modelId,
|
|
49
|
+
reason,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRhZ2VudC51c2FnZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3RzX2FnZW50L3NtYXJ0YWdlbnQudXNhZ2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBYUE7Ozs7O0dBS0c7QUFDSCxNQUFNLE9BQU8sMkJBQTJCO0lBSW5CO0lBQ0E7SUFKWCxJQUFJLEdBQUcsS0FBSyxDQUFDO0lBRXJCLFlBQ21CLEtBQXdDLEVBQ3hDLE1BQW9DO1FBRHBDLFVBQUssR0FBTCxLQUFLLENBQW1DO1FBQ3hDLFdBQU0sR0FBTixNQUFNLENBQThCO0lBQ3BELENBQUM7SUFFSiwyQkFBMkI7SUFDcEIsS0FBSztRQUNWLElBQUksQ0FBQyxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUM7UUFDdkIsSUFBSSxDQUFDLElBQUksR0FBRyxJQUFJLENBQUM7SUFDbkIsQ0FBQztJQUVELGlHQUFpRztJQUMxRixHQUFHLENBQUMsZUFBdUIsRUFBRSxhQUFpQztRQUNuRSxJQUFJLENBQUMsSUFBSSxHQUFHLEtBQUssQ0FBQztRQUNsQixNQUFNLFdBQVcsR0FBRyxhQUFhLENBQUMsV0FBVyxJQUFJLENBQUMsQ0FBQztRQUNuRCxNQUFNLFlBQVksR0FBRyxhQUFhLENBQUMsWUFBWSxJQUFJLENBQUMsQ0FBQztRQUNyRCxNQUFNLEtBQUssR0FBZ0I7WUFDekIsV0FBVztZQUNYLFlBQVk7WUFDWixXQUFXLEVBQUUsV0FBVyxHQUFHLFlBQVk7WUFDdkMsZUFBZSxFQUFFLGFBQWEsQ0FBQyxpQkFBaUIsQ0FBQyxlQUFlLElBQUksQ0FBQztZQUNyRSxnQkFBZ0IsRUFBRSxhQUFhLENBQUMsaUJBQWlCLENBQUMsZ0JBQWdCLElBQUksQ0FBQztTQUN4RSxDQUFDO1FBQ0YsSUFBSSxDQUFDLE1BQU0sQ0FBQztZQUNWLE1BQU0sRUFBRSxVQUFVO1lBQ2xCLFFBQVEsRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLFFBQVE7WUFDN0IsZ0JBQWdCLEVBQUUsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPO1lBQ3BDLGVBQWU7WUFDZixLQUFLO1NBQ04sQ0FBQyxDQUFDO0lBQ0wsQ0FBQztJQUVELCtFQUErRTtJQUN4RSxNQUFNLENBQUMsTUFBdUM7UUFDbkQsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJO1lBQUUsT0FBTztRQUN2QixJQUFJLENBQUMsSUFBSSxHQUFHLEtBQUssQ0FBQztRQUNsQixJQUFJLENBQUMsTUFBTSxDQUFDO1lBQ1YsTUFBTSxFQUFFLFlBQVk7WUFDcEIsUUFBUSxFQUFFLElBQUksQ0FBQyxLQUFLLENBQUMsUUFBUTtZQUM3QixnQkFBZ0IsRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU87WUFDcEMsTUFBTTtTQUNQLENBQUMsQ0FBQztJQUNMLENBQUM7Q0FDRiJ9
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"author": "Task Venture Capital GmbH",
|
|
3
3
|
"license": "MIT",
|
|
4
4
|
"name": "@modelprofile.com/flexharness-agent",
|
|
5
|
-
"version": "8.
|
|
5
|
+
"version": "8.5.0",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"description": "Standalone event-driven agent runtime with generation transactions, context and tool execution contracts.",
|
|
8
8
|
"main": "./dist_ts_agent/index.js",
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
}
|
|
19
19
|
},
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@modelprofile.com/flexharness-models": "8.
|
|
21
|
+
"@modelprofile.com/flexharness-models": "8.5.0",
|
|
22
22
|
"ai": "^7.0.79",
|
|
23
23
|
"zod": "^4.4.1",
|
|
24
24
|
"@noble/hashes": "^2.4.0",
|
package/readme.md
CHANGED
|
@@ -88,7 +88,7 @@ The following Node examples reuse `setup` and, where needed, `tools` from the qu
|
|
|
88
88
|
| `messages` | Current AI SDK message history after projection or compaction. Pass it into another run to continue. |
|
|
89
89
|
| `steps` | Completed model steps, including steps from validation-triggered attempts. A step can call several tools. |
|
|
90
90
|
| `finishReason` | The model's final finish reason; inspect this together with application validation. |
|
|
91
|
-
| `usage` | Input, output and total tokens, plus cache-read and cache-write tokens. |
|
|
91
|
+
| `usage` | Input, output and total tokens, plus cache-read and cache-write tokens, summed over every model call of the run the provider reported: its model steps, retried calls included, and the reported calls of a compaction its context overflow caused. |
|
|
92
92
|
| `toolCalls` | Tool-call IDs, names and inputs, with available outputs or errors. |
|
|
93
93
|
|
|
94
94
|
```typescript
|
|
@@ -154,9 +154,68 @@ Pass these callbacks to `runAgent` or `AgentSession.create()`:
|
|
|
154
154
|
| `onToolCallUpdate(event)` | The call identity and a transient streamed `output`. |
|
|
155
155
|
| `onToolCallFinish(event)` | The call identity plus either `success: true, output` or `success: false, error`. |
|
|
156
156
|
| `onRetry(event)` | Before each wait to retry a model call: `attempt`, `maxAttempts`, `delayMs` and `reason` (`rate_limit`, `overloaded`, `unavailable`). |
|
|
157
|
+
| `onUsage(event)` | Once per model call, as soon as its usage is known; see [Count the usage of every run](#count-the-usage-of-every-run). |
|
|
157
158
|
|
|
158
159
|
Tool updates are transient; the finish callback carries the authoritative final output. Use `subscribe()` for committed session changes (`committed`, `updated`, `archived`), and retain its returned unsubscribe function. Session listeners are delivered in order per listener, have bounded queues and timeouts, and are removed on failure. Streaming callbacks and session-change listeners serve different purposes.
|
|
159
160
|
|
|
161
|
+
## Count the usage of every run
|
|
162
|
+
|
|
163
|
+
A run that throws or is aborted has used tokens too, and its promise carries no result. `onUsage` reports each model call once, as soon as its usage is known, whatever the run's outcome. Sum it to count a run's usage:
|
|
164
|
+
|
|
165
|
+
```typescript
|
|
166
|
+
import type { IAgentUsage } from '@modelprofile.com/flexharness-agent';
|
|
167
|
+
|
|
168
|
+
const used: IAgentUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
|
|
169
|
+
let usageComplete = true;
|
|
170
|
+
try {
|
|
171
|
+
await runAgent({
|
|
172
|
+
...setup,
|
|
173
|
+
tools,
|
|
174
|
+
prompt: 'Convert 10 km to miles.',
|
|
175
|
+
abort: AbortSignal.timeout(60_000),
|
|
176
|
+
onUsage: (event) => {
|
|
177
|
+
if (event.status === 'unreported') {
|
|
178
|
+
usageComplete = false;
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
for (const key of Object.keys(used) as (keyof IAgentUsage)[]) used[key] += event.usage[key];
|
|
182
|
+
},
|
|
183
|
+
});
|
|
184
|
+
} finally {
|
|
185
|
+
console.log(used, usageComplete ? 'complete' : 'lower bound');
|
|
186
|
+
}
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
| Event field | Meaning |
|
|
190
|
+
| --- | --- |
|
|
191
|
+
| `source` | `generation`: a model step of the generation. `compaction`: a model call of a context compaction. |
|
|
192
|
+
| `generationId` | The generation the call belongs to. A compaction carries the generation whose context overflow caused it; a compaction by `compact()` or event retention has none. |
|
|
193
|
+
| `provider`, `requestedModelId` | The model the call was made with: the language model's `provider` and `modelId`. Every event carries both, whatever its status, so key usage caps on them. |
|
|
194
|
+
| `status: 'reported'`, `usage`, `responseModelId` | The provider reported the call's usage. A count the provider leaves out is zero. `responseModelId` is the model id the provider's response named (a provider may answer with a dated model version), or `requestedModelId` when it named none. |
|
|
195
|
+
| `status: 'unreported'`, `reason` | The call ended before the provider reported its usage: `aborted` (the call was aborted), `failed` (the call or its response failed) or `missing` (the response carried no usage). The provider may still have consumed tokens for it; their number is unknown. |
|
|
196
|
+
|
|
197
|
+
When a run returns, its reported calls sum to `result.usage`; count one or the other, not both. A call is reported when the provider's response ends, before its tool calls run, so a run aborted during a tool call still reports the call that requested it. Retried calls and validation retries are included. `onUsage` must not throw; an error it throws is reported like a session listener error and does not change the run's outcome. `runAgent` delivers every event of the run before its promise settles. `AgentSession.create()` accepts the same callback for every generation and compaction of the session; `generate()` and `scheduleGenerate()` without a `transaction` reject as soon as their abort signal fires and may report the interrupted call afterwards; `close()` waits for that report. With a `transaction`, they settle only after every call of the generation has been reported.
|
|
198
|
+
|
|
199
|
+
### What is counted
|
|
200
|
+
|
|
201
|
+
- Every model step of a generation, whatever its outcome, including retried calls and validation retries.
|
|
202
|
+
- The model calls of a context compaction, when the compactor reports them. `contextCompactor` and `onContextOverflow` receive `reportUsage` in their options; it reports into `onUsage` with `source: 'compaction'`. `compactMessages()` from `@modelprofile.com/flexharness/compaction` reports each attempt of its model call when it receives `reportUsage`, so pass the handler's options through:
|
|
203
|
+
|
|
204
|
+
```typescript
|
|
205
|
+
import { compactMessages } from '@modelprofile.com/flexharness/compaction';
|
|
206
|
+
|
|
207
|
+
const session = await AgentSession.create({
|
|
208
|
+
...setup,
|
|
209
|
+
contextCompactor: (messages, _events, options) => compactMessages(setup.model, messages, options),
|
|
210
|
+
onContextOverflow: (messages, options) => compactMessages(setup.model, messages, options),
|
|
211
|
+
onUsage: (event) => console.log(event),
|
|
212
|
+
});
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
A handler that makes its own model calls reports each one through `reportUsage` before its promise settles, a call that fails or is aborted as `unreported`. `AgentModelCallUsageRecorder` does the bookkeeping for one language model: call `start()` when a call begins, `end(responseModelId, usage)` with the AI SDK's `onLanguageModelCallEnd` values, and `settle(reason)` when a call ends otherwise.
|
|
216
|
+
|
|
217
|
+
Not counted: model calls a handler makes without reporting them through `reportUsage`, and model calls outside the session, such as tools that call models themselves. Report those in your own accounting.
|
|
218
|
+
|
|
160
219
|
## Validate an answer and request corrections
|
|
161
220
|
|
|
162
221
|
`validateCompletion` returns `void` to accept a result or a string to add a corrective user message and generate again. `maxValidationRetries` defaults to `0`: a failed validation throws unless retries are configured.
|
|
@@ -243,9 +302,9 @@ Use an application-owned durable adapter for persistent sessions. Its `save(sess
|
|
|
243
302
|
### Bound model context and active events
|
|
244
303
|
|
|
245
304
|
- `contextBuilder({ events })` controls the model-message projection.
|
|
246
|
-
- `contextCompactor(messages, events, { reason, abortSignal })` returns replacement model messages. Provide it to use `session.compact()` or automatic event retention.
|
|
305
|
+
- `contextCompactor(messages, events, { reason, abortSignal, reportUsage })` returns replacement model messages. Provide it to use `session.compact()` or automatic event retention. Report the usage of its model calls through `reportUsage`; see [What is counted](#what-is-counted).
|
|
247
306
|
- `eventRetention: { maxEvents }` triggers compaction and archival when the active event count exceeds the threshold. It also requires an event store with `archive()` support.
|
|
248
|
-
- Context overflow invokes the configured compactor, or the `onContextOverflow` handler. Without either, generation throws `ContextOverflowError`. `maxContextOverflowRetries` defaults to `3`.
|
|
307
|
+
- Context overflow invokes the configured compactor, or the `onContextOverflow(messages, { abortSignal, reportUsage })` handler. Without either, generation throws `ContextOverflowError`. `maxContextOverflowRetries` defaults to `3`.
|
|
249
308
|
- Open transactions and uncertain tool intents constrain when compaction and archival can proceed. Resolve them before manual compaction.
|
|
250
309
|
|
|
251
310
|
Compaction changes the active model context; archival moves covered events out of the active event set. Implement archive retention in your chosen store when you need a complete audit history.
|
|
@@ -42,6 +42,8 @@ import {
|
|
|
42
42
|
type TAgentSessionChangeListener,
|
|
43
43
|
type TAgentCacheSetting,
|
|
44
44
|
type TAgentGenerationPrepare,
|
|
45
|
+
type TAgentModelCallUsageEvent,
|
|
46
|
+
type TAgentModelCallUsageReporter,
|
|
45
47
|
type TAgentPrompt,
|
|
46
48
|
type TAgentToolCallFinishEvent,
|
|
47
49
|
type TAgentToolExecutionReconciliationOptions,
|
|
@@ -51,6 +53,7 @@ import {
|
|
|
51
53
|
validateAgentEventSnapshotV2,
|
|
52
54
|
} from './smartagent.persistence.js';
|
|
53
55
|
import { MAX_RETRY_ATTEMPTS, planModelCallRetry } from './smartagent.retry.js';
|
|
56
|
+
import { AgentModelCallUsageRecorder } from './smartagent.usage.js';
|
|
54
57
|
import type {
|
|
55
58
|
IAgentEventArchive,
|
|
56
59
|
IAgentEventArchiveV2,
|
|
@@ -520,6 +523,19 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
520
523
|
this.platform.reportListenerError(error);
|
|
521
524
|
}
|
|
522
525
|
|
|
526
|
+
private reportModelCallUsage(event: TAgentModelCallUsageEvent): void {
|
|
527
|
+
try {
|
|
528
|
+
this.options.onUsage?.(event);
|
|
529
|
+
} catch (error) {
|
|
530
|
+
this.reportListenerError(error);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/** Reports the calls of a compaction that no generation caused: compact() and event retention. */
|
|
535
|
+
private readonly reportSessionCompactionUsage: TAgentModelCallUsageReporter = (call) => {
|
|
536
|
+
this.reportModelCallUsage({ ...call, source: 'compaction' });
|
|
537
|
+
};
|
|
538
|
+
|
|
523
539
|
private queueNotification(change: IAgentSessionChange, durability: Promise<void>): void {
|
|
524
540
|
const dispatch = (async () => {
|
|
525
541
|
try {
|
|
@@ -1165,6 +1181,7 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
1165
1181
|
|
|
1166
1182
|
private async performCompaction(
|
|
1167
1183
|
reason: 'context-overflow' | 'retention' | 'manual',
|
|
1184
|
+
reportUsage: TAgentModelCallUsageReporter,
|
|
1168
1185
|
abortSignal?: AbortSignal,
|
|
1169
1186
|
): Promise<IContextCompactionEvent | undefined> {
|
|
1170
1187
|
abortSignal?.throwIfAborted();
|
|
@@ -1199,11 +1216,15 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
1199
1216
|
const messages = this.buildContext(visibleCoveredEvents);
|
|
1200
1217
|
const replacementMessages = this.options.contextCompactor
|
|
1201
1218
|
? await this.invokeGenerationCallback(
|
|
1202
|
-
() => this.options.contextCompactor!(
|
|
1219
|
+
() => this.options.contextCompactor!(
|
|
1220
|
+
messages,
|
|
1221
|
+
visibleCoveredEvents,
|
|
1222
|
+
{ abortSignal, reason, reportUsage },
|
|
1223
|
+
),
|
|
1203
1224
|
)
|
|
1204
1225
|
: reason === 'context-overflow' && this.options.onContextOverflow
|
|
1205
1226
|
? await this.invokeGenerationCallback(
|
|
1206
|
-
() => this.options.onContextOverflow!(messages, { abortSignal }),
|
|
1227
|
+
() => this.options.onContextOverflow!(messages, { abortSignal, reportUsage }),
|
|
1207
1228
|
)
|
|
1208
1229
|
: undefined;
|
|
1209
1230
|
if (!replacementMessages) {
|
|
@@ -1261,7 +1282,11 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
1261
1282
|
const combinedAbort = combineAbortSignals([options.abort, this.sessionAbortController.signal]);
|
|
1262
1283
|
const operation = this.queueExclusive(async () => {
|
|
1263
1284
|
combinedAbort.signal.throwIfAborted();
|
|
1264
|
-
await this.performCompaction(
|
|
1285
|
+
await this.performCompaction(
|
|
1286
|
+
options.reason ?? 'manual',
|
|
1287
|
+
this.reportSessionCompactionUsage,
|
|
1288
|
+
combinedAbort.signal,
|
|
1289
|
+
);
|
|
1265
1290
|
}).finally(combinedAbort.cleanup);
|
|
1266
1291
|
return waitWithAbort(operation, [options.abort, this.sessionAbortController.signal]);
|
|
1267
1292
|
}
|
|
@@ -1329,6 +1354,7 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
1329
1354
|
while (this.events.length > retention.maxEvents) {
|
|
1330
1355
|
const compaction = await this.performCompaction(
|
|
1331
1356
|
'retention',
|
|
1357
|
+
this.reportSessionCompactionUsage,
|
|
1332
1358
|
this.sessionAbortController.signal,
|
|
1333
1359
|
);
|
|
1334
1360
|
if (!compaction) return;
|
|
@@ -1706,6 +1732,8 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
1706
1732
|
return this.generateLocked(options);
|
|
1707
1733
|
});
|
|
1708
1734
|
this.generationQueue = run.then(() => undefined, () => undefined);
|
|
1735
|
+
// A transactional generation is not raced against its abort signal: it settles when its
|
|
1736
|
+
// execution has ended, after each of its model calls was reported through `onUsage`.
|
|
1709
1737
|
return options.transaction
|
|
1710
1738
|
? run
|
|
1711
1739
|
: waitWithAbort(run, [options.abort, this.sessionAbortController.signal]);
|
|
@@ -1818,6 +1846,21 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
1818
1846
|
let totalOutput = 0;
|
|
1819
1847
|
let totalCacheRead = 0;
|
|
1820
1848
|
let totalCacheWrite = 0;
|
|
1849
|
+
const recordModelCallUsage = (event: TAgentModelCallUsageEvent): void => {
|
|
1850
|
+
if (event.status === 'reported') {
|
|
1851
|
+
totalInput += event.usage.inputTokens;
|
|
1852
|
+
totalOutput += event.usage.outputTokens;
|
|
1853
|
+
totalCacheRead += event.usage.cacheReadTokens;
|
|
1854
|
+
totalCacheWrite += event.usage.cacheWriteTokens;
|
|
1855
|
+
}
|
|
1856
|
+
this.reportModelCallUsage(event);
|
|
1857
|
+
};
|
|
1858
|
+
const modelCalls = new AgentModelCallUsageRecorder(
|
|
1859
|
+
runtime.model,
|
|
1860
|
+
(call) => recordModelCallUsage({ ...call, source: 'generation', generationId }),
|
|
1861
|
+
);
|
|
1862
|
+
const reportCompactionUsage: TAgentModelCallUsageReporter = (call) =>
|
|
1863
|
+
recordModelCallUsage({ ...call, source: 'compaction', generationId });
|
|
1821
1864
|
let currentInferenceId: string | undefined;
|
|
1822
1865
|
let latestInferenceVisibleIds = new Set(this.events.map((event) => event.id));
|
|
1823
1866
|
const completedInferenceIds = new Set<string>();
|
|
@@ -1892,6 +1935,12 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
1892
1935
|
onError: ({ error }) => {
|
|
1893
1936
|
streamError = error;
|
|
1894
1937
|
},
|
|
1938
|
+
onLanguageModelCallStart: () => {
|
|
1939
|
+
modelCalls.start();
|
|
1940
|
+
},
|
|
1941
|
+
onLanguageModelCallEnd: ({ modelId, usage }) => {
|
|
1942
|
+
modelCalls.end(modelId, usage);
|
|
1943
|
+
},
|
|
1895
1944
|
repairToolCall: async ({ toolCall, tools: availableTools, error }) => {
|
|
1896
1945
|
const lowerName = toolCall.toolName.toLowerCase();
|
|
1897
1946
|
if (
|
|
@@ -2134,10 +2183,6 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
2134
2183
|
completedInferenceIds.add(currentInferenceId);
|
|
2135
2184
|
stepCount++;
|
|
2136
2185
|
}
|
|
2137
|
-
totalInput += step.usage.inputTokens ?? 0;
|
|
2138
|
-
totalOutput += step.usage.outputTokens ?? 0;
|
|
2139
|
-
totalCacheRead += step.usage.inputTokenDetails.cacheReadTokens ?? 0;
|
|
2140
|
-
totalCacheWrite += step.usage.inputTokenDetails.cacheWriteTokens ?? 0;
|
|
2141
2186
|
for (const toolCall of step.toolCalls) {
|
|
2142
2187
|
recordToolCall(toolCalls, toolCallIndexes, toolCall);
|
|
2143
2188
|
}
|
|
@@ -2166,6 +2211,9 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
2166
2211
|
const finishReason = await result.finishReason;
|
|
2167
2212
|
await result.response;
|
|
2168
2213
|
if (streamError) throw streamError;
|
|
2214
|
+
// A stream whose call was aborted can end without an error; its call was aborted, not
|
|
2215
|
+
// answered without usage.
|
|
2216
|
+
modelCalls.settle(combinedAbort.signal.aborted ? 'aborted' : 'missing');
|
|
2169
2217
|
flushReasoning();
|
|
2170
2218
|
await this.flushPersistence();
|
|
2171
2219
|
|
|
@@ -2193,6 +2241,7 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
2193
2241
|
completedResult = generationResult;
|
|
2194
2242
|
return generationResult;
|
|
2195
2243
|
} catch (error) {
|
|
2244
|
+
modelCalls.settle(combinedAbort.signal.aborted ? 'aborted' : 'failed');
|
|
2196
2245
|
flushReasoning();
|
|
2197
2246
|
const effectiveError = streamError ?? error;
|
|
2198
2247
|
if (combinedAbort.signal.aborted) {
|
|
@@ -2252,7 +2301,11 @@ export class AgentSessionEngine implements IAgentSession {
|
|
|
2252
2301
|
}
|
|
2253
2302
|
combinedAbort.signal.throwIfAborted();
|
|
2254
2303
|
contextOverflowRetries++;
|
|
2255
|
-
await this.performCompaction(
|
|
2304
|
+
await this.performCompaction(
|
|
2305
|
+
'context-overflow',
|
|
2306
|
+
reportCompactionUsage,
|
|
2307
|
+
combinedAbort.signal,
|
|
2308
|
+
);
|
|
2256
2309
|
continue;
|
|
2257
2310
|
}
|
|
2258
2311
|
throw effectiveError;
|
package/ts_agent/index.ts
CHANGED
|
@@ -14,6 +14,8 @@ export {
|
|
|
14
14
|
modelMessagesToAgentEvents,
|
|
15
15
|
} from './smartagent.events.js';
|
|
16
16
|
export { ToolRegistry } from './smartagent.classes.toolregistry.js';
|
|
17
|
+
export { AgentModelCallUsageRecorder } from './smartagent.usage.js';
|
|
18
|
+
export type { IAgentModelCallUsageRecorderModel } from './smartagent.usage.js';
|
|
17
19
|
export {
|
|
18
20
|
AgentGenerationLeaseCleanupError,
|
|
19
21
|
ContextOverflowError,
|
|
@@ -48,6 +50,7 @@ export type {
|
|
|
48
50
|
IAgentToolCallRecord,
|
|
49
51
|
IAgentToolCallStartEvent,
|
|
50
52
|
IAgentToolCallUpdateEvent,
|
|
53
|
+
IAgentUsage,
|
|
51
54
|
ProviderOptions,
|
|
52
55
|
TAgentCacheRetention,
|
|
53
56
|
TAgentCacheSetting,
|
|
@@ -55,6 +58,10 @@ export type {
|
|
|
55
58
|
TAgentContextBuilder,
|
|
56
59
|
TAgentContextCompactor,
|
|
57
60
|
TAgentGenerationPrepare,
|
|
61
|
+
TAgentModelCallUnreportedReason,
|
|
62
|
+
TAgentModelCallUsage,
|
|
63
|
+
TAgentModelCallUsageEvent,
|
|
64
|
+
TAgentModelCallUsageReporter,
|
|
58
65
|
TAgentSessionChangeListener,
|
|
59
66
|
TAgentToolCallFinishEvent,
|
|
60
67
|
TAgentToolExecutionReconciliationOptions,
|