@roj-ai/sdk 0.1.31 → 0.1.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/agents/agent.d.ts +5 -1
- package/dist/core/agents/agent.d.ts.map +1 -1
- package/dist/core/agents/agent.js.map +1 -1
- package/dist/core/agents/config.d.ts +5 -1
- package/dist/core/agents/config.d.ts.map +1 -1
- package/dist/core/llm/anthropic.d.ts.map +1 -1
- package/dist/core/llm/anthropic.js +7 -1
- package/dist/core/llm/anthropic.js.map +1 -1
- package/dist/core/llm/cache-breakpoints.d.ts +39 -5
- package/dist/core/llm/cache-breakpoints.d.ts.map +1 -1
- package/dist/core/llm/cache-breakpoints.js +22 -8
- package/dist/core/llm/cache-breakpoints.js.map +1 -1
- package/dist/core/llm/cache-breakpoints.test.js +32 -8
- package/dist/core/llm/cache-breakpoints.test.js.map +1 -1
- package/dist/core/llm/openrouter-mapping.test.js +29 -0
- package/dist/core/llm/openrouter-mapping.test.js.map +1 -1
- package/dist/core/llm/openrouter.d.ts.map +1 -1
- package/dist/core/llm/openrouter.js +28 -8
- package/dist/core/llm/openrouter.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/plugins/result-eviction/plugin.d.ts.map +1 -1
- package/dist/plugins/result-eviction/plugin.js +24 -9
- package/dist/plugins/result-eviction/plugin.js.map +1 -1
- package/dist/plugins/result-eviction/result-eviction.integration.test.js +77 -0
- package/dist/plugins/result-eviction/result-eviction.integration.test.js.map +1 -1
- package/dist/plugins/session-stats/index.d.ts +1 -1
- package/dist/plugins/session-stats/index.d.ts.map +1 -1
- package/dist/plugins/session-stats/plugin.d.ts +16 -0
- package/dist/plugins/session-stats/plugin.d.ts.map +1 -1
- package/dist/plugins/session-stats/plugin.js +29 -2
- package/dist/plugins/session-stats/plugin.js.map +1 -1
- package/dist/plugins/session-stats/plugin.test.d.ts +2 -0
- package/dist/plugins/session-stats/plugin.test.d.ts.map +1 -0
- package/dist/plugins/session-stats/plugin.test.js +145 -0
- package/dist/plugins/session-stats/plugin.test.js.map +1 -0
- package/dist/transport/http/middleware/error-handler.d.ts +1 -1
- package/package.json +2 -2
- package/src/core/agents/agent.ts +5 -2
- package/src/core/agents/config.ts +5 -1
- package/src/core/llm/anthropic.ts +7 -1
- package/src/core/llm/cache-breakpoints.test.ts +38 -8
- package/src/core/llm/cache-breakpoints.ts +54 -9
- package/src/core/llm/openrouter-mapping.test.ts +37 -1
- package/src/core/llm/openrouter.ts +35 -9
- package/src/index.ts +2 -1
- package/src/plugins/result-eviction/plugin.ts +26 -11
- package/src/plugins/result-eviction/result-eviction.integration.test.ts +88 -0
- package/src/plugins/session-stats/index.ts +1 -1
- package/src/plugins/session-stats/plugin.test.ts +160 -0
- package/src/plugins/session-stats/plugin.ts +48 -2
|
@@ -1,21 +1,28 @@
|
|
|
1
1
|
import { agentEvents } from '../../core/agents/state.js';
|
|
2
|
+
import { contextEvents } from '../../core/context/state.js';
|
|
2
3
|
import { llmEvents } from '../../core/llm/state.js';
|
|
3
4
|
import { definePlugin } from '../../core/plugins/plugin-builder.js';
|
|
4
5
|
import { selectPluginState } from '../../core/sessions/reducer.js';
|
|
5
6
|
import { toolEvents } from '../../core/tools/state.js';
|
|
7
|
+
import { userChatEvents } from '../../plugins/user-chat/plugin.js';
|
|
6
8
|
const createInitialStats = () => ({
|
|
7
9
|
totalTokens: 0,
|
|
8
10
|
promptTokens: 0,
|
|
9
11
|
completionTokens: 0,
|
|
12
|
+
cacheReadTokens: 0,
|
|
13
|
+
cacheWriteTokens: 0,
|
|
10
14
|
totalCost: 0,
|
|
11
15
|
llmCalls: 0,
|
|
12
16
|
llmErrors: 0,
|
|
13
17
|
toolCalls: 0,
|
|
14
18
|
toolErrors: 0,
|
|
19
|
+
userMessages: 0,
|
|
20
|
+
compactions: 0,
|
|
15
21
|
agentCount: 0,
|
|
16
22
|
firstEventAt: null,
|
|
17
23
|
lastEventAt: null,
|
|
18
24
|
byProvider: {},
|
|
25
|
+
byAgent: {},
|
|
19
26
|
});
|
|
20
27
|
const PLUGIN_STATE_KEY = 'sessionStats';
|
|
21
28
|
/**
|
|
@@ -28,11 +35,11 @@ export function selectSessionStats(sessionState) {
|
|
|
28
35
|
// Plugin
|
|
29
36
|
// ============================================================================
|
|
30
37
|
export const sessionStatsPlugin = definePlugin('session-stats')
|
|
31
|
-
.events([agentEvents, llmEvents, toolEvents])
|
|
38
|
+
.events([agentEvents, llmEvents, toolEvents, contextEvents, userChatEvents])
|
|
32
39
|
.state({
|
|
33
40
|
key: PLUGIN_STATE_KEY,
|
|
34
41
|
initial: createInitialStats,
|
|
35
|
-
reduce: (stats, event) => {
|
|
42
|
+
reduce: (stats, event, sessionState) => {
|
|
36
43
|
const now = event.timestamp;
|
|
37
44
|
const withTimestamp = (partial) => ({
|
|
38
45
|
...stats,
|
|
@@ -61,13 +68,29 @@ export const sessionStatsPlugin = definePlugin('session-stats')
|
|
|
61
68
|
},
|
|
62
69
|
}
|
|
63
70
|
: stats.byProvider;
|
|
71
|
+
// Core reducer runs before plugin slices, so the spawning agent is already in state.
|
|
72
|
+
const definitionName = sessionState.agents.get(event.agentId)?.definitionName;
|
|
73
|
+
const byAgent = definitionName
|
|
74
|
+
? {
|
|
75
|
+
...stats.byAgent,
|
|
76
|
+
[definitionName]: {
|
|
77
|
+
llmCalls: (stats.byAgent[definitionName]?.llmCalls ?? 0) + 1,
|
|
78
|
+
promptTokens: (stats.byAgent[definitionName]?.promptTokens ?? 0) + event.metrics.promptTokens,
|
|
79
|
+
completionTokens: (stats.byAgent[definitionName]?.completionTokens ?? 0) + event.metrics.completionTokens,
|
|
80
|
+
totalCost: (stats.byAgent[definitionName]?.totalCost ?? 0) + (event.metrics.cost ?? 0),
|
|
81
|
+
},
|
|
82
|
+
}
|
|
83
|
+
: stats.byAgent;
|
|
64
84
|
return withTimestamp({
|
|
65
85
|
llmCalls: stats.llmCalls + 1,
|
|
66
86
|
totalTokens: stats.totalTokens + event.metrics.totalTokens,
|
|
67
87
|
promptTokens: stats.promptTokens + event.metrics.promptTokens,
|
|
68
88
|
completionTokens: stats.completionTokens + event.metrics.completionTokens,
|
|
89
|
+
cacheReadTokens: stats.cacheReadTokens + (event.metrics.cachedTokens ?? 0),
|
|
90
|
+
cacheWriteTokens: stats.cacheWriteTokens + (event.metrics.cacheWriteTokens ?? 0),
|
|
69
91
|
totalCost: stats.totalCost + (event.metrics.cost ?? 0),
|
|
70
92
|
byProvider,
|
|
93
|
+
byAgent,
|
|
71
94
|
});
|
|
72
95
|
}
|
|
73
96
|
case 'inference_failed':
|
|
@@ -76,6 +99,10 @@ export const sessionStatsPlugin = definePlugin('session-stats')
|
|
|
76
99
|
return withTimestamp({ toolCalls: stats.toolCalls + 1 });
|
|
77
100
|
case 'tool_failed':
|
|
78
101
|
return withTimestamp({ toolErrors: stats.toolErrors + 1 });
|
|
102
|
+
case 'user_chat_message_received':
|
|
103
|
+
return withTimestamp({ userMessages: stats.userMessages + 1 });
|
|
104
|
+
case 'context_compacted':
|
|
105
|
+
return withTimestamp({ compactions: stats.compactions + 1 });
|
|
79
106
|
default:
|
|
80
107
|
return stats;
|
|
81
108
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../../../src/plugins/session-stats/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAA;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAE9D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;
|
|
1
|
+
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../../../src/plugins/session-stats/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAA;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAE9D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AA+C9D,MAAM,kBAAkB,GAAG,GAAsB,EAAE,CAAC,CAAC;IACpD,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,CAAC;IACf,gBAAgB,EAAE,CAAC;IACnB,eAAe,EAAE,CAAC;IAClB,gBAAgB,EAAE,CAAC;IACnB,SAAS,EAAE,CAAC;IACZ,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,YAAY,EAAE,CAAC;IACf,WAAW,EAAE,CAAC;IACd,UAAU,EAAE,CAAC;IACb,YAAY,EAAE,IAAI;IAClB,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,EAAE;IACd,OAAO,EAAE,EAAE;CACX,CAAC,CAAA;AAEF,MAAM,gBAAgB,GAAG,cAAc,CAAA;AAEvC;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,YAA0B;IAC5D,OAAO,iBAAiB,CAAoB,YAAY,EAAE,gBAAgB,CAAC,IAAI,kBAAkB,EAAE,CAAA;AACpG,CAAC;AAED,+EAA+E;AAC/E,SAAS;AACT,+EAA+E;AAE/E,MAAM,CAAC,MAAM,kBAAkB,GAAG,YAAY,CAAC,eAAe,CAAC;KAC7D,MAAM,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;KAC3E,KAAK,CAAC;IACN,GAAG,EAAE,gBAAgB;IACrB,OAAO,EAAE,kBAAkB;IAC3B,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE;QACtC,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAA;QAC3B,MAAM,aAAa,GAAG,CAAC,OAAmC,EAAqB,EAAE,CAAC,CAAC;YAClF,GAAG,KAAK;YACR,GAAG,OAAO;YACV,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,GAAG;YACvC,WAAW,EAAE,GAAG;SAChB,CAAC,CAAA;QAEF,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,eAAe;gBACnB,OAAO,aAAa,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC,CAAA;YAE3D,0EAA0E;YAC1E,oEAAoE;YACpE,sDAAsD;YACtD,KAAK,qBAAqB,CAAC;YAC3B,KAAK,+BAA+B,CAAC,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAA;gBACvC,MAAM,UAAU,GAAG,QAAQ;oBAC1B,CAAC,CAAC;wBACD,GAAG,KAAK,CAAC,UAAU;wBACnB,CAAC,QAAQ,CAAC,EAAE;4BACX,QAAQ,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC;4BACzD,WAAW,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW;4BACvF,YAAY,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY;4BAC1F,gBAAgB,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,gBAAgB,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB;4BACtG,SAAS,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;yBACnF;qBACD;oBACD,CAAC,CAAC,KAAK,CAAC,UAAU,CAAA;gBACnB,qFAAqF;gBACrF,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,cAAc,CAAA;gBAC7E,MAAM,OAAO,GAAG,cAAc;oBAC7B,CAAC,CAAC;wBACD,GAAG,KAAK,CAAC,OAAO;wBAChB,CAAC,cAAc,CAAC,EAAE;4BACjB,QAAQ,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC;4BAC5D,YAAY,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY;4BAC7F,gBAAgB,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,gBAAgB,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB;4BACzG,SAAS,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;yBACtF;qBACD;oBACD,CAAC,CAAC,KAAK,CAAC,OAAO,CAAA;gBAChB,OAAO,aAAa,CAAC;oBACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC;oBAC5B,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW;oBAC1D,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY;oBAC7D,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB;oBACzE,eAAe,EAAE,KAAK,CAAC,eAAe,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC;oBAC1E,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC;oBAChF,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;oBACtD,UAAU;oBACV,OAAO;iBACP,CAAC,CAAA;YACH,CAAC;YAED,KAAK,kBAAkB;gBACtB,OAAO,aAAa,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC,CAAA;YAEzD,KAAK,cAAc;gBAClB,OAAO,aAAa,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC,CAAA;YAEzD,KAAK,aAAa;gBACjB,OAAO,aAAa,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC,CAAA;YAE3D,KAAK,4BAA4B;gBAChC,OAAO,aAAa,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC,CAAA;YAE/D,KAAK,mBAAmB;gBACvB,OAAO,aAAa,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC,CAAA;YAE7D;gBACC,OAAO,KAAK,CAAA;QACd,CAAC;IACF,CAAC;CACD,CAAC;KACD,KAAK,EAAE,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugin.test.d.ts","sourceRoot":"","sources":["../../../src/plugins/session-stats/plugin.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
import { AgentId } from '../../core/agents/schema.js';
|
|
3
|
+
import { agentEvents } from '../../core/agents/state.js';
|
|
4
|
+
import { contextEvents } from '../../core/context/state.js';
|
|
5
|
+
import { withSessionId } from '../../core/events/test-helpers.js';
|
|
6
|
+
import { llmEvents } from '../../core/llm/state.js';
|
|
7
|
+
import { createApplyEvent } from '../../core/sessions/apply-event.js';
|
|
8
|
+
import { SessionId } from '../../core/sessions/schema.js';
|
|
9
|
+
import { createSessionState } from '../../core/sessions/state.js';
|
|
10
|
+
import { ToolCallId } from '../../core/tools/schema.js';
|
|
11
|
+
import { toolEvents } from '../../core/tools/state.js';
|
|
12
|
+
import { userChatEvents } from '../../plugins/user-chat/plugin.js';
|
|
13
|
+
import { ChatMessageId } from '../../plugins/user-chat/schema.js';
|
|
14
|
+
import { selectSessionStats, sessionStatsPlugin } from './plugin.js';
|
|
15
|
+
const sessionId = SessionId('session-stats-test');
|
|
16
|
+
const orchestrator = AgentId('orchestrator-1');
|
|
17
|
+
const worker = AgentId('worker-1');
|
|
18
|
+
const applyEvent = createApplyEvent([sessionStatsPlugin.create()]);
|
|
19
|
+
const metrics = (overrides = {}) => ({
|
|
20
|
+
promptTokens: 100,
|
|
21
|
+
completionTokens: 20,
|
|
22
|
+
totalTokens: 120,
|
|
23
|
+
latencyMs: 10,
|
|
24
|
+
model: 'test-model',
|
|
25
|
+
provider: 'anthropic',
|
|
26
|
+
cost: 0.5,
|
|
27
|
+
...overrides,
|
|
28
|
+
});
|
|
29
|
+
const spawn = (agentId, definitionName) => withSessionId(sessionId, agentEvents.create('agent_spawned', { agentId, definitionName, parentId: null }));
|
|
30
|
+
const inference = (agentId, m) => withSessionId(sessionId, llmEvents.create('inference_completed', {
|
|
31
|
+
agentId,
|
|
32
|
+
consumedMessageIds: [],
|
|
33
|
+
response: { content: 'ok', toolCalls: [] },
|
|
34
|
+
metrics: m,
|
|
35
|
+
}));
|
|
36
|
+
const reduce = (events) => selectSessionStats(events.reduce(applyEvent, createSessionState(sessionId, 'test-preset', 0)));
|
|
37
|
+
describe('session-stats plugin', () => {
|
|
38
|
+
it('starts from zeroed counters', () => {
|
|
39
|
+
const stats = reduce([]);
|
|
40
|
+
expect(stats.userMessages).toBe(0);
|
|
41
|
+
expect(stats.compactions).toBe(0);
|
|
42
|
+
expect(stats.cacheReadTokens).toBe(0);
|
|
43
|
+
expect(stats.cacheWriteTokens).toBe(0);
|
|
44
|
+
expect(stats.byAgent).toEqual({});
|
|
45
|
+
});
|
|
46
|
+
it('counts user chat messages', () => {
|
|
47
|
+
const stats = reduce([
|
|
48
|
+
spawn(orchestrator, 'orchestrator'),
|
|
49
|
+
withSessionId(sessionId, userChatEvents.create('user_chat_message_received', {
|
|
50
|
+
agentId: orchestrator,
|
|
51
|
+
messageId: ChatMessageId('m1'),
|
|
52
|
+
content: 'hello',
|
|
53
|
+
timestamp: 1,
|
|
54
|
+
})),
|
|
55
|
+
withSessionId(sessionId, userChatEvents.create('user_chat_message_received', {
|
|
56
|
+
agentId: orchestrator,
|
|
57
|
+
messageId: ChatMessageId('m2'),
|
|
58
|
+
content: 'again',
|
|
59
|
+
timestamp: 2,
|
|
60
|
+
})),
|
|
61
|
+
]);
|
|
62
|
+
expect(stats.userMessages).toBe(2);
|
|
63
|
+
});
|
|
64
|
+
it('counts context compactions', () => {
|
|
65
|
+
const stats = reduce([
|
|
66
|
+
spawn(orchestrator, 'orchestrator'),
|
|
67
|
+
withSessionId(sessionId, contextEvents.create('context_compacted', {
|
|
68
|
+
agentId: orchestrator,
|
|
69
|
+
compactedContent: 'summary',
|
|
70
|
+
newConversationHistory: [],
|
|
71
|
+
originalTokens: 1000,
|
|
72
|
+
compactedTokens: 100,
|
|
73
|
+
messagesRemoved: 8,
|
|
74
|
+
})),
|
|
75
|
+
]);
|
|
76
|
+
expect(stats.compactions).toBe(1);
|
|
77
|
+
});
|
|
78
|
+
it('sums cache read and write tokens across main and auxiliary inferences', () => {
|
|
79
|
+
const stats = reduce([
|
|
80
|
+
spawn(orchestrator, 'orchestrator'),
|
|
81
|
+
inference(orchestrator, metrics({ cachedTokens: 800, cacheWriteTokens: 200 })),
|
|
82
|
+
withSessionId(sessionId, llmEvents.create('auxiliary_inference_completed', {
|
|
83
|
+
agentId: orchestrator,
|
|
84
|
+
metrics: metrics({ cachedTokens: 50, cacheWriteTokens: 10 }),
|
|
85
|
+
})),
|
|
86
|
+
// Providers that report no cache usage must not break the totals.
|
|
87
|
+
inference(orchestrator, metrics()),
|
|
88
|
+
]);
|
|
89
|
+
expect(stats.cacheReadTokens).toBe(850);
|
|
90
|
+
expect(stats.cacheWriteTokens).toBe(210);
|
|
91
|
+
expect(stats.llmCalls).toBe(3);
|
|
92
|
+
});
|
|
93
|
+
it('breaks LLM usage down by agent definition name', () => {
|
|
94
|
+
const stats = reduce([
|
|
95
|
+
spawn(orchestrator, 'orchestrator'),
|
|
96
|
+
spawn(worker, 'coder'),
|
|
97
|
+
inference(orchestrator, metrics({ promptTokens: 100, completionTokens: 20, cost: 0.5 })),
|
|
98
|
+
inference(worker, metrics({ promptTokens: 300, completionTokens: 40, cost: 1.25 })),
|
|
99
|
+
inference(worker, metrics({ promptTokens: 200, completionTokens: 10, cost: 0.25 })),
|
|
100
|
+
withSessionId(sessionId, llmEvents.create('auxiliary_inference_completed', {
|
|
101
|
+
agentId: worker,
|
|
102
|
+
metrics: metrics({ promptTokens: 10, completionTokens: 1, cost: 0.05 }),
|
|
103
|
+
})),
|
|
104
|
+
]);
|
|
105
|
+
expect(stats.byAgent).toEqual({
|
|
106
|
+
orchestrator: { llmCalls: 1, promptTokens: 100, completionTokens: 20, totalCost: 0.5 },
|
|
107
|
+
coder: { llmCalls: 3, promptTokens: 510, completionTokens: 51, totalCost: 1.55 },
|
|
108
|
+
});
|
|
109
|
+
expect(stats.totalCost).toBeCloseTo(2.05, 10);
|
|
110
|
+
});
|
|
111
|
+
it('keeps existing counters working', () => {
|
|
112
|
+
const stats = reduce([
|
|
113
|
+
spawn(orchestrator, 'orchestrator'),
|
|
114
|
+
inference(orchestrator, metrics()),
|
|
115
|
+
withSessionId(sessionId, llmEvents.create('inference_failed', { agentId: orchestrator, error: 'boom' })),
|
|
116
|
+
withSessionId(sessionId, toolEvents.create('tool_started', {
|
|
117
|
+
agentId: orchestrator,
|
|
118
|
+
toolCallId: ToolCallId('tc-1'),
|
|
119
|
+
toolName: 'read_file',
|
|
120
|
+
input: {},
|
|
121
|
+
})),
|
|
122
|
+
withSessionId(sessionId, toolEvents.create('tool_failed', {
|
|
123
|
+
agentId: orchestrator,
|
|
124
|
+
toolCallId: ToolCallId('tc-1'),
|
|
125
|
+
error: 'ENOENT',
|
|
126
|
+
})),
|
|
127
|
+
]);
|
|
128
|
+
expect(stats.agentCount).toBe(1);
|
|
129
|
+
expect(stats.llmCalls).toBe(1);
|
|
130
|
+
expect(stats.llmErrors).toBe(1);
|
|
131
|
+
expect(stats.toolCalls).toBe(1);
|
|
132
|
+
expect(stats.toolErrors).toBe(1);
|
|
133
|
+
expect(stats.totalTokens).toBe(120);
|
|
134
|
+
expect(stats.promptTokens).toBe(100);
|
|
135
|
+
expect(stats.completionTokens).toBe(20);
|
|
136
|
+
expect(stats.byProvider.anthropic).toEqual({
|
|
137
|
+
llmCalls: 1,
|
|
138
|
+
totalTokens: 120,
|
|
139
|
+
promptTokens: 100,
|
|
140
|
+
completionTokens: 20,
|
|
141
|
+
totalCost: 0.5,
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
//# sourceMappingURL=plugin.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugin.test.js","sourceRoot":"","sources":["../../../src/plugins/session-stats/plugin.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,UAAU,CAAA;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAG7D,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAA;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAEpE,MAAM,SAAS,GAAG,SAAS,CAAC,oBAAoB,CAAC,CAAA;AACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;AAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;AAElC,MAAM,UAAU,GAAG,gBAAgB,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;AAElE,MAAM,OAAO,GAAG,CAAC,YAAiC,EAAE,EAAc,EAAE,CAAC,CAAC;IACrE,YAAY,EAAE,GAAG;IACjB,gBAAgB,EAAE,EAAE;IACpB,WAAW,EAAE,GAAG;IAChB,SAAS,EAAE,EAAE;IACb,KAAK,EAAE,YAAY;IACnB,QAAQ,EAAE,WAAW;IACrB,IAAI,EAAE,GAAG;IACT,GAAG,SAAS;CACZ,CAAC,CAAA;AAEF,MAAM,KAAK,GAAG,CAAC,OAAgB,EAAE,cAAsB,EAAE,EAAE,CAC1D,aAAa,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,eAAe,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;AAE3G,MAAM,SAAS,GAAG,CAAC,OAAgB,EAAE,CAAa,EAAE,EAAE,CACrD,aAAa,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,qBAAqB,EAAE;IAChE,OAAO;IACP,kBAAkB,EAAE,EAAE;IACtB,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE;IAC1C,OAAO,EAAE,CAAC;CACV,CAAC,CAAC,CAAA;AAEJ,MAAM,MAAM,GAAG,CAAC,MAAqB,EAAE,EAAE,CAAC,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,kBAAkB,CAAC,SAAS,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAExI,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,CAAA;QACxB,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC;YACpB,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;YACnC,aAAa,CAAC,SAAS,EAAE,cAAc,CAAC,MAAM,CAAC,4BAA4B,EAAE;gBAC5E,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC;gBAC9B,OAAO,EAAE,OAAO;gBAChB,SAAS,EAAE,CAAC;aACZ,CAAC,CAAC;YACH,aAAa,CAAC,SAAS,EAAE,cAAc,CAAC,MAAM,CAAC,4BAA4B,EAAE;gBAC5E,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC;gBAC9B,OAAO,EAAE,OAAO;gBAChB,SAAS,EAAE,CAAC;aACZ,CAAC,CAAC;SACH,CAAC,CAAA;QACF,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,KAAK,GAAG,MAAM,CAAC;YACpB,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;YACnC,aAAa,CAAC,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,mBAAmB,EAAE;gBAClE,OAAO,EAAE,YAAY;gBACrB,gBAAgB,EAAE,SAAS;gBAC3B,sBAAsB,EAAE,EAAE;gBAC1B,cAAc,EAAE,IAAI;gBACpB,eAAe,EAAE,GAAG;gBACpB,eAAe,EAAE,CAAC;aAClB,CAAC,CAAC;SACH,CAAC,CAAA;QACF,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAClC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uEAAuE,EAAE,GAAG,EAAE;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC;YACpB,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;YACnC,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9E,aAAa,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,+BAA+B,EAAE;gBAC1E,OAAO,EAAE,YAAY;gBACrB,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC;aAC5D,CAAC,CAAC;YACH,kEAAkE;YAClE,SAAS,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;SAClC,CAAC,CAAA;QACF,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACvC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACzD,MAAM,KAAK,GAAG,MAAM,CAAC;YACpB,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;YACnC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC;YACtB,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;YACxF,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACnF,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACnF,aAAa,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,+BAA+B,EAAE;gBAC1E,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACvE,CAAC,CAAC;SACH,CAAC,CAAA;QACF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YAC7B,YAAY,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE;YACtF,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;SAChF,CAAC,CAAA;QACF,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC;YACpB,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;YACnC,SAAS,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;YAClC,aAAa,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACxG,aAAa,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,cAAc,EAAE;gBAC1D,OAAO,EAAE,YAAY;gBACrB,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC;gBAC9B,QAAQ,EAAE,WAAW;gBACrB,KAAK,EAAE,EAAE;aACT,CAAC,CAAC;YACH,aAAa,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE;gBACzD,OAAO,EAAE,YAAY;gBACrB,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC;gBAC9B,KAAK,EAAE,QAAQ;aACf,CAAC,CAAC;SACH,CAAC,CAAA;QACF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAChC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC9B,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC/B,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC/B,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAChC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACnC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACvC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC;YAC1C,QAAQ,EAAE,CAAC;YACX,WAAW,EAAE,GAAG;YAChB,YAAY,EAAE,GAAG;YACjB,gBAAgB,EAAE,EAAE;YACpB,SAAS,EAAE,GAAG;SACd,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}
|
|
@@ -17,5 +17,5 @@ export declare const errorHandler: (error: Error, c: Context<AppEnv>) => Respons
|
|
|
17
17
|
type: string;
|
|
18
18
|
message: string;
|
|
19
19
|
};
|
|
20
|
-
}, 100 | 500 | 404 | 409 | 400 | 403 | -1 | 429 | 401 | 200 | 300 | 501 | 201 | 102 | 103 | 202 | 203 | 206 | 207 | 208 | 226 | 301 | 302 | 303 | 305 | 306 | 307 | 308 | 402 | 405 | 406 | 407 | 408 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 421 | 422 | 423 | 424 | 425 | 426 | 428 | 431 | 451 | 502 | 503 | 504 | 505 | 506 | 507 | 508 |
|
|
20
|
+
}, 100 | 500 | 404 | 409 | 400 | 403 | -1 | 429 | 401 | 200 | 300 | 501 | 201 | 510 | 102 | 103 | 202 | 203 | 206 | 207 | 208 | 226 | 301 | 302 | 303 | 305 | 306 | 307 | 308 | 402 | 405 | 406 | 407 | 408 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 421 | 422 | 423 | 424 | 425 | 426 | 428 | 431 | 451 | 502 | 503 | 504 | 505 | 506 | 507 | 508 | 511, "json">;
|
|
21
21
|
//# sourceMappingURL=error-handler.d.ts.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@roj-ai/sdk",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.33",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -135,7 +135,7 @@
|
|
|
135
135
|
"type-check": "tsc --noEmit"
|
|
136
136
|
},
|
|
137
137
|
"dependencies": {
|
|
138
|
-
"@roj-ai/transport": "^0.1.
|
|
138
|
+
"@roj-ai/transport": "^0.1.33",
|
|
139
139
|
"@hono/zod-validator": "0.7.6",
|
|
140
140
|
"hono": "4.12.5",
|
|
141
141
|
"ignore": "7.0.5",
|
package/src/core/agents/agent.ts
CHANGED
|
@@ -14,7 +14,7 @@ import type { AgentState, HandlerResult } from '~/core/agents/state.js'
|
|
|
14
14
|
import { agentEvents } from '~/core/agents/state.js'
|
|
15
15
|
import { withSessionId } from '~/core/events/test-helpers.js'
|
|
16
16
|
import type { FileStore } from '~/core/file-store/types.js'
|
|
17
|
-
import { applyCacheBreakpoint } from '~/core/llm/cache-breakpoints.js'
|
|
17
|
+
import { applyCacheBreakpoint, type CacheTtlConfig } from '~/core/llm/cache-breakpoints.js'
|
|
18
18
|
import type { ToolResultContent } from '~/core/llm/llm-log-types.js'
|
|
19
19
|
import type { InferenceRequest, InferenceResponse, LLMError, LLMMessage, LLMProvider } from '~/core/llm/provider.js'
|
|
20
20
|
import { LLMCallId, ModelId } from '~/core/llm/schema.js'
|
|
@@ -87,8 +87,11 @@ export interface AgentConfig<TInput = unknown> {
|
|
|
87
87
|
* — useful for long-lived agents (e.g. an orchestrator that waits minutes
|
|
88
88
|
* between user turns) where the default 5-minute TTL would expire and force
|
|
89
89
|
* full re-uploads. Omit (or '5m') for the standard tier.
|
|
90
|
+
*
|
|
91
|
+
* A bare tier applies to both breakpoints; `{ prefix, tail }` sets them
|
|
92
|
+
* independently, which usually pays better. See {@link CacheTtlConfig}.
|
|
90
93
|
*/
|
|
91
|
-
cacheTtl?:
|
|
94
|
+
cacheTtl?: CacheTtlConfig
|
|
92
95
|
/** Outer backoff between error-resume cycles (see BaseAgentConfig.errorResumeBackoff). */
|
|
93
96
|
errorResumeBackoff?: { baseDelayMs?: number; maxDelayMs?: number }
|
|
94
97
|
}
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import z from 'zod/v4'
|
|
6
6
|
import type { DebounceCallback } from '~/core/agents/debounce.js'
|
|
7
|
+
import type { CacheTtlConfig } from '~/core/llm/cache-breakpoints.js'
|
|
7
8
|
import type { LLMMiddleware } from '~/core/llm/middleware.js'
|
|
8
9
|
import { ModelId } from '~/core/llm/schema.js'
|
|
9
10
|
import type { AgentPluginConfig } from '~/core/plugins/plugin-builder.js'
|
|
@@ -57,8 +58,11 @@ export interface BaseAgentConfig<TInput = unknown> {
|
|
|
57
58
|
* '1h' opts into Anthropic's extended cache tier for long-lived agents
|
|
58
59
|
* (e.g. an orchestrator that waits minutes between user turns). Defaults
|
|
59
60
|
* to the standard 5-minute tier.
|
|
61
|
+
*
|
|
62
|
+
* A bare tier applies to both breakpoints; `{ prefix, tail }` sets them
|
|
63
|
+
* independently, which usually pays better. See {@link CacheTtlConfig}.
|
|
60
64
|
*/
|
|
61
|
-
cacheTtl?:
|
|
65
|
+
cacheTtl?: CacheTtlConfig
|
|
62
66
|
/**
|
|
63
67
|
* Outer backoff between error-resume cycles after an inference exhausts its
|
|
64
68
|
* LLM retries on a retryable error (the message stays queued and is retried).
|
|
@@ -326,10 +326,16 @@ export class AnthropicProvider implements RoutableLLMProvider {
|
|
|
326
326
|
const mappedMessages = await Promise.all(sanitizedMessages.map((m) => this.mapMessage(m, context)))
|
|
327
327
|
const mergedMessages = this.mergeConsecutiveMessages(mappedMessages)
|
|
328
328
|
|
|
329
|
+
// The system prompt precedes every message and Anthropic requires a longer
|
|
330
|
+
// TTL to come BEFORE a shorter one, so it has to match the longest TTL any
|
|
331
|
+
// breakpoint asks for — otherwise the order is 5m → 1h → 5m.
|
|
332
|
+
const systemCacheControl: AnthropicCacheControl = { type: 'ephemeral' }
|
|
333
|
+
if (sanitizedMessages.some((m) => m.cacheControl?.ttl === '1h')) systemCacheControl.ttl = '1h'
|
|
334
|
+
|
|
329
335
|
const body: AnthropicRequestBody = {
|
|
330
336
|
model: request.model ?? this.defaultModel,
|
|
331
337
|
max_tokens: request.maxTokens ?? 100_000,
|
|
332
|
-
system: [{ type: 'text', text: request.systemPrompt, cache_control:
|
|
338
|
+
system: [{ type: 'text', text: request.systemPrompt, cache_control: systemCacheControl }],
|
|
333
339
|
messages: mergedMessages,
|
|
334
340
|
stream: false,
|
|
335
341
|
}
|
|
@@ -37,14 +37,44 @@ describe('applyCacheBreakpoint', () => {
|
|
|
37
37
|
expect(cachedIndices(result)).toEqual([3])
|
|
38
38
|
})
|
|
39
39
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
40
|
+
// prefix = 1, tail = 5 for userMessages(6) with suffix 0 and cachedPrefixCount 2
|
|
41
|
+
const ttlsOf = (messages: LLMMessage[]) => [1, 5].map((idx) => {
|
|
42
|
+
const m = messages[idx]
|
|
43
|
+
return 'cacheControl' in m && m.cacheControl ? m.cacheControl.ttl : undefined
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
test('a bare tier applies to both breakpoints', () => {
|
|
47
|
+
const result = applyCacheBreakpoint(userMessages(6), 0, '1h', 2)
|
|
48
|
+
expect(ttlsOf(result)).toEqual(['1h', '1h'])
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
test('{ prefix } leaves the tail on the default tier', () => {
|
|
52
|
+
// Usually the paying configuration for an interactive agent: the immutable
|
|
53
|
+
// preamble survives human pauses, the churning tail stays cheap to write.
|
|
54
|
+
const result = applyCacheBreakpoint(userMessages(6), 0, { prefix: '1h' }, 2)
|
|
55
|
+
expect(ttlsOf(result)).toEqual(['1h', undefined])
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
test('{ prefix, tail } sets each breakpoint independently', () => {
|
|
59
|
+
const result = applyCacheBreakpoint(userMessages(6), 0, { prefix: '1h', tail: '5m' }, 2)
|
|
60
|
+
expect(ttlsOf(result)).toEqual(['1h', '5m'])
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
test('{ tail } alone is honoured — the SDK does not second-guess the caller', () => {
|
|
64
|
+
const result = applyCacheBreakpoint(userMessages(6), 0, { tail: '1h' }, 2)
|
|
65
|
+
expect(ttlsOf(result)).toEqual([undefined, '1h'])
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
test('prefixIdx === tailIdx → the single mark covers the preamble, so it takes the prefix ttl', () => {
|
|
69
|
+
const result = applyCacheBreakpoint(userMessages(3), 0, { prefix: '1h' }, 3)
|
|
70
|
+
expect(cachedIndices(result)).toEqual([2])
|
|
71
|
+
const only = result[2]
|
|
72
|
+
expect('cacheControl' in only && only.cacheControl).toEqual({ type: 'ephemeral', ttl: '1h' })
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
test('no ttl → neither breakpoint carries one', () => {
|
|
76
|
+
const result = applyCacheBreakpoint(userMessages(6), 0, undefined, 2)
|
|
77
|
+
expect(ttlsOf(result)).toEqual([undefined, undefined])
|
|
48
78
|
})
|
|
49
79
|
|
|
50
80
|
test('does not mutate the input array', () => {
|
|
@@ -1,5 +1,44 @@
|
|
|
1
1
|
import type { LLMMessage, LLMMessageCacheControl } from '~/core/agents/state.js'
|
|
2
2
|
|
|
3
|
+
/** Anthropic's two ephemeral cache tiers: 5-minute (write 1.25×) and 1-hour (write 2×). Reads are 0.1× on both. */
|
|
4
|
+
export type CacheTtl = '5m' | '1h'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Per-breakpoint prompt cache TTL.
|
|
8
|
+
*
|
|
9
|
+
* A bare tier applies to both breakpoints. The object form sets them
|
|
10
|
+
* independently, which is usually what you want: the stable prefix and the
|
|
11
|
+
* churning conversation tail have opposite cost profiles.
|
|
12
|
+
*
|
|
13
|
+
* Choosing between them is an economic question about a specific agent's call
|
|
14
|
+
* pattern, so the SDK does not pick for you:
|
|
15
|
+
*
|
|
16
|
+
* - A long TTL on the **prefix** pays off when calls are spread further apart
|
|
17
|
+
* than the short tier survives — an orchestrator waiting on a human.
|
|
18
|
+
* - A long TTL on the **tail** costs 2× on every write while the tail is
|
|
19
|
+
* rewritten turn-to-turn anyway, so it only pays when consecutive calls are
|
|
20
|
+
* themselves minutes apart.
|
|
21
|
+
*
|
|
22
|
+
* Worked example, from a measured 5-week production session: 95% of inferences
|
|
23
|
+
* followed a sub-5-minute gap, so `'1h'` uniformly came out ~2.5% *worse* than
|
|
24
|
+
* the default, while `{ prefix: '1h' }` targeted the 29 rewrites that actually
|
|
25
|
+
* expired. A batch agent called in a tight loop would want the opposite.
|
|
26
|
+
*
|
|
27
|
+
* Constraint: Anthropic requires a longer TTL to precede a shorter one, and the
|
|
28
|
+
* prefix breakpoint always sits before the tail. So `{ prefix: '5m', tail: '1h' }`
|
|
29
|
+
* is invalid at the API and will be rejected — the tail TTL must not exceed the
|
|
30
|
+
* prefix TTL.
|
|
31
|
+
*/
|
|
32
|
+
export type CacheTtlConfig = CacheTtl | { prefix?: CacheTtl; tail?: CacheTtl }
|
|
33
|
+
|
|
34
|
+
const resolveTtls = (config: CacheTtlConfig | undefined): { prefix?: CacheTtl; tail?: CacheTtl } => {
|
|
35
|
+
if (config === undefined) return {}
|
|
36
|
+
if (typeof config === 'string') return { prefix: config, tail: config }
|
|
37
|
+
return config
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const cacheControl = (ttl: CacheTtl | undefined): LLMMessageCacheControl => ttl ? { type: 'ephemeral', ttl } : { type: 'ephemeral' }
|
|
41
|
+
|
|
3
42
|
/**
|
|
4
43
|
* Mark the prompt cache breakpoints on a message list.
|
|
5
44
|
*
|
|
@@ -26,22 +65,26 @@ import type { LLMMessage, LLMMessageCacheControl } from '~/core/agents/state.js'
|
|
|
26
65
|
* every inference AND compaction call. Anthropic allows up to 4 breakpoints
|
|
27
66
|
* and matches the longest cached prefix, so the two coexist.
|
|
28
67
|
*
|
|
29
|
-
* If the prefix and tail indices coincide, only one breakpoint is set
|
|
68
|
+
* If the prefix and tail indices coincide, only one breakpoint is set — it
|
|
69
|
+
* covers the preamble, so it takes the prefix TTL.
|
|
30
70
|
*
|
|
31
|
-
* `ttl`
|
|
32
|
-
*
|
|
33
|
-
*
|
|
71
|
+
* `ttl` sets the cache tier per breakpoint; see {@link CacheTtlConfig} for how
|
|
72
|
+
* to choose. Providers must additionally give the system block the longest TTL
|
|
73
|
+
* in use, since it precedes every message and Anthropic rejects a longer entry
|
|
74
|
+
* that follows a shorter one.
|
|
34
75
|
*/
|
|
35
76
|
export function applyCacheBreakpoint(
|
|
36
77
|
messages: LLMMessage[],
|
|
37
78
|
uncachedSuffixCount: number,
|
|
38
|
-
ttl?:
|
|
79
|
+
ttl?: CacheTtlConfig,
|
|
39
80
|
cachedPrefixCount = 0,
|
|
40
81
|
): LLMMessage[] {
|
|
41
|
-
const
|
|
82
|
+
const ttls = resolveTtls(ttl)
|
|
83
|
+
const prefixCacheControl = cacheControl(ttls.prefix)
|
|
84
|
+
const tailCacheControl = cacheControl(ttls.tail)
|
|
42
85
|
const result = [...messages]
|
|
43
86
|
|
|
44
|
-
const mark = (idx: number) => {
|
|
87
|
+
const mark = (idx: number, cacheControl: LLMMessageCacheControl) => {
|
|
45
88
|
if (idx < 0 || idx >= result.length) return
|
|
46
89
|
const target = result[idx]
|
|
47
90
|
switch (target.role) {
|
|
@@ -59,7 +102,9 @@ export function applyCacheBreakpoint(
|
|
|
59
102
|
// summary, sliding recent window), so the immutable preamble is read at 0.1×
|
|
60
103
|
// on every inference AND compaction call instead of being re-billed each turn.
|
|
61
104
|
const prefixIdx = cachedPrefixCount - 1
|
|
62
|
-
if (prefixIdx !== tailIdx) mark(prefixIdx)
|
|
63
|
-
|
|
105
|
+
if (prefixIdx !== tailIdx) mark(prefixIdx, prefixCacheControl)
|
|
106
|
+
// When the two coincide there is only one entry to pin, and it covers the
|
|
107
|
+
// preamble — so it takes the prefix TTL.
|
|
108
|
+
mark(tailIdx, prefixIdx === tailIdx ? prefixCacheControl : tailCacheControl)
|
|
64
109
|
return result
|
|
65
110
|
}
|
|
@@ -21,7 +21,7 @@ const createProvider = () =>
|
|
|
21
21
|
defaultModel: 'anthropic/claude-haiku-4.5',
|
|
22
22
|
})
|
|
23
23
|
|
|
24
|
-
type AnyBlock = { type: string; cache_control?: { type: 'ephemeral' }; [key: string]: unknown }
|
|
24
|
+
type AnyBlock = { type: string; cache_control?: { type: 'ephemeral'; ttl?: '5m' | '1h' }; [key: string]: unknown }
|
|
25
25
|
type OpenRouterMessage = {
|
|
26
26
|
role: string
|
|
27
27
|
content: string | AnyBlock[]
|
|
@@ -152,6 +152,42 @@ describe('OpenRouterProvider buildHttpRequest', () => {
|
|
|
152
152
|
expect(last.content).toBe('<session-context>plugin status</session-context>')
|
|
153
153
|
})
|
|
154
154
|
|
|
155
|
+
test('ttl reaches the wire and the system block matches it', async () => {
|
|
156
|
+
// The whole point of the fix: OpenRouter dropped `ttl` on the floor, so a
|
|
157
|
+
// preset asking for the 1-hour tier silently ran on the 5-minute default.
|
|
158
|
+
const messages = applyCacheBreakpoint([
|
|
159
|
+
{ role: 'user', content: 'preamble' },
|
|
160
|
+
{ role: 'assistant', content: 'ok' },
|
|
161
|
+
{ role: 'user', content: 'work' },
|
|
162
|
+
], 0, { prefix: '1h' }, 1)
|
|
163
|
+
|
|
164
|
+
const http = await createProvider().buildHttpRequest(buildRequest(messages))
|
|
165
|
+
const msgs = getBodyMessages(http.body)
|
|
166
|
+
|
|
167
|
+
// System precedes every message, so it must carry the longest TTL in use —
|
|
168
|
+
// Anthropic rejects a longer entry that follows a shorter one.
|
|
169
|
+
const systemBlocks = asBlocks(msgs[0].content)
|
|
170
|
+
expect(systemBlocks[0].cache_control).toEqual({ type: 'ephemeral', ttl: '1h' })
|
|
171
|
+
|
|
172
|
+
// prefix breakpoint (message index 0 → body index 1) carries 1h
|
|
173
|
+
expect(asBlocks(msgs[1].content)[0].cache_control).toEqual({ type: 'ephemeral', ttl: '1h' })
|
|
174
|
+
// tail breakpoint stays on the cheap short tier
|
|
175
|
+
expect(asBlocks(msgs[3].content)[0].cache_control).toEqual({ type: 'ephemeral' })
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
test('no ttl requested: system block stays on the default tier', async () => {
|
|
179
|
+
const messages = applyCacheBreakpoint([
|
|
180
|
+
{ role: 'user', content: 'preamble' },
|
|
181
|
+
{ role: 'user', content: 'work' },
|
|
182
|
+
], 0, undefined, 1)
|
|
183
|
+
|
|
184
|
+
const http = await createProvider().buildHttpRequest(buildRequest(messages))
|
|
185
|
+
const msgs = getBodyMessages(http.body)
|
|
186
|
+
|
|
187
|
+
expect(asBlocks(msgs[0].content)[0].cache_control).toEqual({ type: 'ephemeral' })
|
|
188
|
+
expect(asBlocks(msgs[1].content)[0].cache_control).toEqual({ type: 'ephemeral' })
|
|
189
|
+
})
|
|
190
|
+
|
|
155
191
|
test('no flag on any message: only system has cache_control', async () => {
|
|
156
192
|
const http = await createProvider().buildHttpRequest(buildRequest([
|
|
157
193
|
{ role: 'user', content: 'What is 2+2?' },
|
|
@@ -89,11 +89,17 @@ interface OpenRouterErrorResponse {
|
|
|
89
89
|
// Request body types
|
|
90
90
|
// ============================================================================
|
|
91
91
|
|
|
92
|
+
interface OpenRouterCacheControl {
|
|
93
|
+
type: 'ephemeral'
|
|
94
|
+
/** Omitted = the provider default 5-minute tier. */
|
|
95
|
+
ttl?: '5m' | '1h'
|
|
96
|
+
}
|
|
97
|
+
|
|
92
98
|
interface OpenRouterContentItem {
|
|
93
99
|
type: string
|
|
94
100
|
text?: string
|
|
95
101
|
image_url?: { url: string; detail?: string }
|
|
96
|
-
cache_control?:
|
|
102
|
+
cache_control?: OpenRouterCacheControl
|
|
97
103
|
}
|
|
98
104
|
|
|
99
105
|
interface OpenRouterMessage {
|
|
@@ -104,18 +110,34 @@ interface OpenRouterMessage {
|
|
|
104
110
|
}
|
|
105
111
|
|
|
106
112
|
/**
|
|
107
|
-
* Add `cache_control
|
|
108
|
-
*
|
|
109
|
-
*
|
|
113
|
+
* Add `cache_control` to the LAST content block of an OpenRouterMessage,
|
|
114
|
+
* converting string content to an array text block first so the mark has a
|
|
115
|
+
* place to live.
|
|
116
|
+
*
|
|
117
|
+
* `ttl` reaches Anthropic through the Chat Completions surface. It is NOT
|
|
118
|
+
* expressible on the Responses API, where a breakpoint is downgraded to the
|
|
119
|
+
* default 5-minute tier — so this must stay on `/chat/completions`.
|
|
110
120
|
*/
|
|
111
|
-
function applyCacheControlToLastBlock(msg: OpenRouterMessage): void {
|
|
121
|
+
function applyCacheControlToLastBlock(msg: OpenRouterMessage, cacheControl: OpenRouterCacheControl): void {
|
|
112
122
|
if (typeof msg.content === 'string') {
|
|
113
|
-
msg.content = [{ type: 'text', text: msg.content, cache_control:
|
|
123
|
+
msg.content = [{ type: 'text', text: msg.content, cache_control: cacheControl }]
|
|
114
124
|
return
|
|
115
125
|
}
|
|
116
126
|
if (msg.content.length === 0) return
|
|
117
127
|
const lastIdx = msg.content.length - 1
|
|
118
|
-
msg.content[lastIdx] = { ...msg.content[lastIdx], cache_control:
|
|
128
|
+
msg.content[lastIdx] = { ...msg.content[lastIdx], cache_control: cacheControl }
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Longest TTL requested anywhere in the message list, or undefined when none.
|
|
133
|
+
*
|
|
134
|
+
* The system prompt precedes every message, and Anthropic requires a longer TTL
|
|
135
|
+
* to come BEFORE a shorter one. So when any breakpoint asks for 1h, the system
|
|
136
|
+
* block has to carry 1h too — otherwise the order is 5m → 1h → 5m and the
|
|
137
|
+
* longer entry is invalid.
|
|
138
|
+
*/
|
|
139
|
+
function longestCacheTtl(messages: LLMMessage[]): '1h' | undefined {
|
|
140
|
+
return messages.some((m) => m.cacheControl?.ttl === '1h') ? '1h' : undefined
|
|
119
141
|
}
|
|
120
142
|
|
|
121
143
|
interface OpenRouterToolDefinition {
|
|
@@ -261,8 +283,10 @@ export class OpenRouterProvider implements LLMProvider {
|
|
|
261
283
|
const sanitizedMessages = sanitizeProviderMessages(request.messages, this.name, this.logger)
|
|
262
284
|
const mappedMessages = await Promise.all(sanitizedMessages.map((m) => this.mapMessage(m, context)))
|
|
263
285
|
|
|
286
|
+
const systemTtl = longestCacheTtl(sanitizedMessages)
|
|
287
|
+
const systemCacheControl: OpenRouterCacheControl = systemTtl ? { type: 'ephemeral', ttl: systemTtl } : { type: 'ephemeral' }
|
|
264
288
|
const messages: OpenRouterMessage[] = [
|
|
265
|
-
{ role: 'system', content: [{ type: 'text', text: request.systemPrompt, cache_control:
|
|
289
|
+
{ role: 'system', content: [{ type: 'text', text: request.systemPrompt, cache_control: systemCacheControl }] },
|
|
266
290
|
...mappedMessages,
|
|
267
291
|
]
|
|
268
292
|
|
|
@@ -317,7 +341,9 @@ export class OpenRouterProvider implements LLMProvider {
|
|
|
317
341
|
private async mapMessage(msg: LLMMessage, context?: InferenceContext): Promise<OpenRouterMessage> {
|
|
318
342
|
const mapped = await this.mapMessageContent(msg, context)
|
|
319
343
|
if (msg.cacheControl) {
|
|
320
|
-
|
|
344
|
+
const cacheControl: OpenRouterCacheControl = { type: 'ephemeral' }
|
|
345
|
+
if (msg.cacheControl.ttl) cacheControl.ttl = msg.cacheControl.ttl
|
|
346
|
+
applyCacheControlToLastBlock(mapped, cacheControl)
|
|
321
347
|
}
|
|
322
348
|
return mapped
|
|
323
349
|
}
|