@artemiskit/adapter-trueforge 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/client.d.ts +19 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8892 -0
- package/dist/mapper.d.ts +7 -0
- package/dist/mapper.d.ts.map +1 -0
- package/dist/types.d.ts +104 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +35 -0
- package/src/client.test.ts +790 -0
- package/src/client.ts +333 -0
- package/src/index.ts +7 -0
- package/src/mapper.ts +221 -0
- package/src/test-preload.ts +4 -0
- package/src/types.ts +146 -0
- package/tsconfig.json +13 -0
|
@@ -0,0 +1,790 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
import type { AgentTask } from '@artemiskit/core';
|
|
3
|
+
import { TrueForgeAdapter } from './client';
|
|
4
|
+
import { normalizeTrueForgeActions, sanitizeTrueForgeEvents } from './mapper';
|
|
5
|
+
import { type TrueForgeClient, createLingProviderSetup } from './types';
|
|
6
|
+
|
|
7
|
+
const TASK: AgentTask = {
|
|
8
|
+
id: 'scenario-repair',
|
|
9
|
+
fixturePath: '/fixtures/scenario-repair',
|
|
10
|
+
allowedPaths: ['scenario.yaml'],
|
|
11
|
+
allowedTools: ['workspace_read', 'workspace_patch', 'workspace_run'],
|
|
12
|
+
maxActions: 8,
|
|
13
|
+
timeoutMs: 1_000,
|
|
14
|
+
acceptanceCommands: ['akit validate scenario.yaml'],
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const TURN_CREATED = {
|
|
18
|
+
type: 'turn.created' as const,
|
|
19
|
+
id: 'event-1',
|
|
20
|
+
turnId: 'turn-1',
|
|
21
|
+
previousTurnId: null,
|
|
22
|
+
state: { status: 'running' as const },
|
|
23
|
+
createdAt: '2026-08-21T10:00:00.000Z',
|
|
24
|
+
threadId: null,
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const TURN_DONE = {
|
|
28
|
+
type: 'turn.done' as const,
|
|
29
|
+
id: 'event-4',
|
|
30
|
+
threadId: null,
|
|
31
|
+
createdAt: '2026-08-21T10:00:01.000Z',
|
|
32
|
+
state: {
|
|
33
|
+
status: 'done' as const,
|
|
34
|
+
output: null,
|
|
35
|
+
requiredActions: [],
|
|
36
|
+
completedAt: '2026-08-21T10:00:01.000Z',
|
|
37
|
+
metrics: { totalInputTokens: 20, totalOutputTokens: 5, totalTokens: 25 },
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
function createClient(events = [TURN_CREATED, TURN_DONE]) {
|
|
42
|
+
const calls = {
|
|
43
|
+
sessions: [] as unknown[],
|
|
44
|
+
turns: [] as unknown[],
|
|
45
|
+
cancellations: [] as string[],
|
|
46
|
+
providers: [] as unknown[],
|
|
47
|
+
mcpServers: [] as unknown[],
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
const client: TrueForgeClient = {
|
|
51
|
+
settings: {
|
|
52
|
+
modelProviders: {
|
|
53
|
+
async createOrUpdate(request) {
|
|
54
|
+
calls.providers.push(request);
|
|
55
|
+
return { data: { name: request.manifest.type } };
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
mcpServers: {
|
|
59
|
+
async createOrUpdate(request) {
|
|
60
|
+
calls.mcpServers.push(request);
|
|
61
|
+
return { data: { name: request.manifest.name } };
|
|
62
|
+
},
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
sessions: {
|
|
66
|
+
async create(request) {
|
|
67
|
+
calls.sessions.push(request);
|
|
68
|
+
return { data: { id: 'session-1' } };
|
|
69
|
+
},
|
|
70
|
+
async createTurnStream(sessionId, request) {
|
|
71
|
+
calls.turns.push({ sessionId, request });
|
|
72
|
+
return (async function* () {
|
|
73
|
+
for (const event of events) yield event;
|
|
74
|
+
})();
|
|
75
|
+
},
|
|
76
|
+
async cancel(sessionId) {
|
|
77
|
+
calls.cancellations.push(sessionId);
|
|
78
|
+
return {};
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
return { client, calls };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
describe('TrueForge event mapping', () => {
|
|
87
|
+
it('maps correlated MCP calls to semantic actions and redacts retained content', () => {
|
|
88
|
+
const events = [
|
|
89
|
+
TURN_CREATED,
|
|
90
|
+
{
|
|
91
|
+
type: 'model.message' as const,
|
|
92
|
+
id: 'event-2',
|
|
93
|
+
threadId: 'main',
|
|
94
|
+
createdAt: '2026-08-21T10:00:00.100Z',
|
|
95
|
+
toolCalls: [
|
|
96
|
+
{
|
|
97
|
+
id: 'call-1',
|
|
98
|
+
type: 'function' as const,
|
|
99
|
+
function: { name: 'workspace_run', arguments: '{"command":"akit validate"}' },
|
|
100
|
+
providerSpecificFields: {
|
|
101
|
+
authorization: 'fixture-sensitive-value',
|
|
102
|
+
authToken: 'auth-token-fixture',
|
|
103
|
+
clientSecret: 'client-secret-fixture',
|
|
104
|
+
credentials: { value: 'credentials-fixture' },
|
|
105
|
+
inputTokens: 12,
|
|
106
|
+
'private-key-pem': 'private-key-fixture',
|
|
107
|
+
},
|
|
108
|
+
toolInfo: {
|
|
109
|
+
type: 'mcp' as const,
|
|
110
|
+
serverId: 'server-1',
|
|
111
|
+
serverName: 'sandbox',
|
|
112
|
+
name: 'workspace_run',
|
|
113
|
+
},
|
|
114
|
+
},
|
|
115
|
+
],
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
type: 'tool.response' as const,
|
|
119
|
+
id: 'event-3',
|
|
120
|
+
threadId: 'main',
|
|
121
|
+
toolCallId: 'call-1',
|
|
122
|
+
content:
|
|
123
|
+
'{"exitCode":0,"stdout":"token=fixture-sensitive-value","contact":"owner@example.com"}',
|
|
124
|
+
createdAt: '2026-08-21T10:00:00.350Z',
|
|
125
|
+
},
|
|
126
|
+
TURN_DONE,
|
|
127
|
+
];
|
|
128
|
+
const originalEvents = structuredClone(events);
|
|
129
|
+
|
|
130
|
+
expect(normalizeTrueForgeActions(events, ['fixture-sensitive-value'])).toEqual([
|
|
131
|
+
{
|
|
132
|
+
type: 'command',
|
|
133
|
+
name: 'workspace_run',
|
|
134
|
+
status: 'success',
|
|
135
|
+
durationMs: 250,
|
|
136
|
+
summary: '{"exitCode":0,"stdout":"[REDACTED]","contact":"[REDACTED]"}',
|
|
137
|
+
},
|
|
138
|
+
]);
|
|
139
|
+
const sanitized = sanitizeTrueForgeEvents(events, ['fixture-sensitive-value']);
|
|
140
|
+
expect(JSON.stringify(sanitized)).not.toContain('fixture-sensitive-value');
|
|
141
|
+
expect(JSON.stringify(sanitized)).not.toContain('owner@example.com');
|
|
142
|
+
expect(JSON.stringify(sanitized)).not.toContain('auth-token-fixture');
|
|
143
|
+
expect(JSON.stringify(sanitized)).not.toContain('client-secret-fixture');
|
|
144
|
+
expect(JSON.stringify(sanitized)).not.toContain('credentials-fixture');
|
|
145
|
+
expect(JSON.stringify(sanitized)).not.toContain('private-key-fixture');
|
|
146
|
+
expect(
|
|
147
|
+
(sanitized[1].toolCalls as Array<{ providerSpecificFields: Record<string, unknown> }>)[0]
|
|
148
|
+
.providerSpecificFields.inputTokens
|
|
149
|
+
).toBe(12);
|
|
150
|
+
expect(events).toEqual(originalEvents);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it('normalizes qualified tool names and marks non-zero command exits as errors', () => {
|
|
154
|
+
const events = [
|
|
155
|
+
{
|
|
156
|
+
type: 'model.message' as const,
|
|
157
|
+
id: 'event-1',
|
|
158
|
+
threadId: 'main',
|
|
159
|
+
createdAt: '2026-08-21T10:00:00.000Z',
|
|
160
|
+
toolCalls: [
|
|
161
|
+
{
|
|
162
|
+
id: 'call-1',
|
|
163
|
+
type: 'function' as const,
|
|
164
|
+
function: { name: 'sandbox__workspace_run', arguments: '{}' },
|
|
165
|
+
toolInfo: {
|
|
166
|
+
type: 'mcp' as const,
|
|
167
|
+
serverId: 'server-1',
|
|
168
|
+
serverName: 'sandbox',
|
|
169
|
+
name: 'sandbox__workspace_run',
|
|
170
|
+
},
|
|
171
|
+
},
|
|
172
|
+
],
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
type: 'tool.response' as const,
|
|
176
|
+
id: 'event-2',
|
|
177
|
+
threadId: 'main',
|
|
178
|
+
toolCallId: 'call-1',
|
|
179
|
+
content: '{"exitCode":2,"stderr":"validation failed"}',
|
|
180
|
+
createdAt: '2026-08-21T10:00:00.010Z',
|
|
181
|
+
},
|
|
182
|
+
];
|
|
183
|
+
|
|
184
|
+
expect(normalizeTrueForgeActions(events)).toEqual([
|
|
185
|
+
{
|
|
186
|
+
type: 'command',
|
|
187
|
+
name: 'workspace_run',
|
|
188
|
+
status: 'error',
|
|
189
|
+
durationMs: 10,
|
|
190
|
+
summary: '{"exitCode":2,"stderr":"validation failed"}',
|
|
191
|
+
},
|
|
192
|
+
]);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it('merges SDK message deltas before correlating tool responses without mutating events', () => {
|
|
196
|
+
const events = [
|
|
197
|
+
{
|
|
198
|
+
type: 'model.message' as const,
|
|
199
|
+
id: 'message-1',
|
|
200
|
+
threadId: 'main',
|
|
201
|
+
createdAt: '2026-08-21T10:00:00.000Z',
|
|
202
|
+
content: '',
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
type: 'model.message.delta' as const,
|
|
206
|
+
id: 'message-1',
|
|
207
|
+
threadId: 'main',
|
|
208
|
+
createdAt: '2026-08-21T10:00:00.005Z',
|
|
209
|
+
toolCalls: [
|
|
210
|
+
{
|
|
211
|
+
index: 0,
|
|
212
|
+
id: 'call-delta',
|
|
213
|
+
type: 'function' as const,
|
|
214
|
+
function: { name: 'sandbox__workspace_read', arguments: '{"path":"scenario.yaml"}' },
|
|
215
|
+
toolInfo: {
|
|
216
|
+
type: 'mcp' as const,
|
|
217
|
+
serverId: 'server-1',
|
|
218
|
+
serverName: 'sandbox',
|
|
219
|
+
name: 'sandbox__workspace_read',
|
|
220
|
+
},
|
|
221
|
+
},
|
|
222
|
+
],
|
|
223
|
+
},
|
|
224
|
+
{
|
|
225
|
+
type: 'tool.response' as const,
|
|
226
|
+
id: 'event-response',
|
|
227
|
+
threadId: 'main',
|
|
228
|
+
toolCallId: 'call-delta',
|
|
229
|
+
content: '{"content":"scenario"}',
|
|
230
|
+
createdAt: '2026-08-21T10:00:00.020Z',
|
|
231
|
+
},
|
|
232
|
+
];
|
|
233
|
+
const originalEvents = structuredClone(events);
|
|
234
|
+
|
|
235
|
+
expect(normalizeTrueForgeActions(events)).toEqual([
|
|
236
|
+
{
|
|
237
|
+
type: 'file',
|
|
238
|
+
name: 'workspace_read',
|
|
239
|
+
status: 'success',
|
|
240
|
+
durationMs: 20,
|
|
241
|
+
summary: '{"content":"scenario"}',
|
|
242
|
+
},
|
|
243
|
+
]);
|
|
244
|
+
expect(events).toEqual(originalEvents);
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
it('detects nested MCP command failures in structured content', () => {
|
|
248
|
+
const events = [
|
|
249
|
+
{
|
|
250
|
+
type: 'model.message' as const,
|
|
251
|
+
id: 'event-1',
|
|
252
|
+
threadId: 'main',
|
|
253
|
+
createdAt: '2026-08-21T10:00:00.000Z',
|
|
254
|
+
toolCalls: [
|
|
255
|
+
{
|
|
256
|
+
id: 'call-1',
|
|
257
|
+
type: 'function' as const,
|
|
258
|
+
function: { name: 'workspace_run', arguments: '{}' },
|
|
259
|
+
toolInfo: {
|
|
260
|
+
type: 'mcp' as const,
|
|
261
|
+
serverId: 'server-1',
|
|
262
|
+
serverName: 'sandbox',
|
|
263
|
+
name: 'workspace_run',
|
|
264
|
+
},
|
|
265
|
+
},
|
|
266
|
+
],
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
type: 'tool.response' as const,
|
|
270
|
+
id: 'event-2',
|
|
271
|
+
threadId: 'main',
|
|
272
|
+
toolCallId: 'call-1',
|
|
273
|
+
content: '{"structuredContent":{"exitCode":7,"stderr":"invalid scenario"}}',
|
|
274
|
+
createdAt: '2026-08-21T10:00:00.010Z',
|
|
275
|
+
},
|
|
276
|
+
];
|
|
277
|
+
|
|
278
|
+
expect(normalizeTrueForgeActions(events)[0]?.status).toBe('error');
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
it('preserves a top-level structured tool rejection', () => {
|
|
282
|
+
const events = [
|
|
283
|
+
{
|
|
284
|
+
type: 'model.message' as const,
|
|
285
|
+
id: 'event-1',
|
|
286
|
+
threadId: 'main',
|
|
287
|
+
createdAt: '2026-08-21T10:00:00.000Z',
|
|
288
|
+
toolCalls: [
|
|
289
|
+
{
|
|
290
|
+
id: 'call-1',
|
|
291
|
+
type: 'function' as const,
|
|
292
|
+
function: { name: 'workspace_patch', arguments: '{}' },
|
|
293
|
+
toolInfo: {
|
|
294
|
+
type: 'mcp' as const,
|
|
295
|
+
serverId: 'server-1',
|
|
296
|
+
serverName: 'sandbox',
|
|
297
|
+
name: 'workspace_patch',
|
|
298
|
+
},
|
|
299
|
+
},
|
|
300
|
+
],
|
|
301
|
+
},
|
|
302
|
+
{
|
|
303
|
+
type: 'tool.response' as const,
|
|
304
|
+
id: 'event-2',
|
|
305
|
+
threadId: 'main',
|
|
306
|
+
toolCallId: 'call-1',
|
|
307
|
+
content: '{"status":"rejected","reason":"path denied"}',
|
|
308
|
+
createdAt: '2026-08-21T10:00:00.010Z',
|
|
309
|
+
},
|
|
310
|
+
];
|
|
311
|
+
|
|
312
|
+
expect(normalizeTrueForgeActions(events)[0]?.status).toBe('rejected');
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
it('preserves a nested structured tool rejection', () => {
|
|
316
|
+
const events = [
|
|
317
|
+
{
|
|
318
|
+
type: 'model.message' as const,
|
|
319
|
+
id: 'event-1',
|
|
320
|
+
threadId: 'main',
|
|
321
|
+
createdAt: '2026-08-21T10:00:00.000Z',
|
|
322
|
+
toolCalls: [
|
|
323
|
+
{
|
|
324
|
+
id: 'call-1',
|
|
325
|
+
type: 'function' as const,
|
|
326
|
+
function: { name: 'workspace_patch', arguments: '{}' },
|
|
327
|
+
toolInfo: {
|
|
328
|
+
type: 'mcp' as const,
|
|
329
|
+
serverId: 'server-1',
|
|
330
|
+
serverName: 'sandbox',
|
|
331
|
+
name: 'workspace_patch',
|
|
332
|
+
},
|
|
333
|
+
},
|
|
334
|
+
],
|
|
335
|
+
},
|
|
336
|
+
{
|
|
337
|
+
type: 'tool.response' as const,
|
|
338
|
+
id: 'event-2',
|
|
339
|
+
threadId: 'main',
|
|
340
|
+
toolCallId: 'call-1',
|
|
341
|
+
content: '{"structuredContent":{"status":"rejected","reason":"tool denied"}}',
|
|
342
|
+
createdAt: '2026-08-21T10:00:00.010Z',
|
|
343
|
+
},
|
|
344
|
+
];
|
|
345
|
+
|
|
346
|
+
expect(normalizeTrueForgeActions(events)[0]?.status).toBe('rejected');
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
it('preserves sandbox denials serialized inside TrueForge error content', () => {
|
|
350
|
+
const events = [
|
|
351
|
+
{
|
|
352
|
+
type: 'model.message' as const,
|
|
353
|
+
id: 'event-1',
|
|
354
|
+
threadId: 'main',
|
|
355
|
+
createdAt: '2026-08-21T10:00:00.000Z',
|
|
356
|
+
toolCalls: [
|
|
357
|
+
{
|
|
358
|
+
id: 'call-1',
|
|
359
|
+
type: 'function' as const,
|
|
360
|
+
function: { name: 'workspace_run', arguments: '{"command":"ls"}' },
|
|
361
|
+
toolInfo: {
|
|
362
|
+
type: 'mcp' as const,
|
|
363
|
+
serverId: 'server-1',
|
|
364
|
+
serverName: 'sandbox',
|
|
365
|
+
name: 'workspace_run',
|
|
366
|
+
},
|
|
367
|
+
},
|
|
368
|
+
],
|
|
369
|
+
},
|
|
370
|
+
{
|
|
371
|
+
type: 'tool.response' as const,
|
|
372
|
+
id: 'event-2',
|
|
373
|
+
threadId: 'main',
|
|
374
|
+
toolCallId: 'call-1',
|
|
375
|
+
content:
|
|
376
|
+
'{"error":[{"type":"text","text":"{\\"ok\\":false,\\"error\\":{\\"code\\":\\"SANDBOX_COMMAND_DENIED\\",\\"message\\":\\"SANDBOX_COMMAND_DENIED\\"}}"}]}',
|
|
377
|
+
createdAt: '2026-08-21T10:00:00.010Z',
|
|
378
|
+
},
|
|
379
|
+
];
|
|
380
|
+
|
|
381
|
+
expect(normalizeTrueForgeActions(events)[0]?.status).toBe('rejected');
|
|
382
|
+
});
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
describe('TrueForgeAdapter', () => {
|
|
386
|
+
it('builds a Ling-compatible custom provider and model reference', () => {
|
|
387
|
+
expect(createLingProviderSetup()).toEqual({
|
|
388
|
+
model: { name: 'ling/ling-3-flash' },
|
|
389
|
+
provider: {
|
|
390
|
+
type: 'custom',
|
|
391
|
+
name: 'ling',
|
|
392
|
+
baseUrl: 'https://api.ant-ling.com/v1',
|
|
393
|
+
models: [{ modelId: 'Ling-3.0-flash', name: 'ling-3-flash', properties: {} }],
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
it('runs a session and streamed turn without mutating settings', async () => {
|
|
399
|
+
const { client, calls } = createClient();
|
|
400
|
+
const adapter = new TrueForgeAdapter(
|
|
401
|
+
{
|
|
402
|
+
agent: {
|
|
403
|
+
model: { name: 'ling/ling-3-flash' },
|
|
404
|
+
mcpServers: [{ name: 'sandbox', enableTools: ['@all'], requireApprovalForTools: [] }],
|
|
405
|
+
},
|
|
406
|
+
buildPrompt: (task) => `Repair the fixture at ${task.fixturePath}`,
|
|
407
|
+
collectOutcome: async () => ({
|
|
408
|
+
acceptancePassed: true,
|
|
409
|
+
changedPaths: ['scenario.yaml'],
|
|
410
|
+
finalDiff: 'diff --git a/scenario.yaml b/scenario.yaml',
|
|
411
|
+
}),
|
|
412
|
+
},
|
|
413
|
+
client
|
|
414
|
+
);
|
|
415
|
+
|
|
416
|
+
const outcome = await adapter.run(TASK);
|
|
417
|
+
|
|
418
|
+
expect(calls.providers).toEqual([]);
|
|
419
|
+
expect(calls.mcpServers).toEqual([]);
|
|
420
|
+
expect(calls.sessions).toEqual([
|
|
421
|
+
{
|
|
422
|
+
agent: {
|
|
423
|
+
spec: {
|
|
424
|
+
model: { name: 'ling/ling-3-flash' },
|
|
425
|
+
mcpServers: [{ name: 'sandbox', enableTools: ['@all'], requireApprovalForTools: [] }],
|
|
426
|
+
},
|
|
427
|
+
},
|
|
428
|
+
},
|
|
429
|
+
]);
|
|
430
|
+
expect(calls.turns).toEqual([
|
|
431
|
+
{
|
|
432
|
+
sessionId: 'session-1',
|
|
433
|
+
request: {
|
|
434
|
+
input: [
|
|
435
|
+
{ type: 'user.message', content: 'Repair the fixture at /fixtures/scenario-repair' },
|
|
436
|
+
],
|
|
437
|
+
},
|
|
438
|
+
},
|
|
439
|
+
]);
|
|
440
|
+
expect(outcome).toMatchObject({
|
|
441
|
+
taskId: 'scenario-repair',
|
|
442
|
+
completed: true,
|
|
443
|
+
acceptancePassed: true,
|
|
444
|
+
sessionId: 'session-1',
|
|
445
|
+
turnId: 'turn-1',
|
|
446
|
+
metrics: { totalTokens: 25 },
|
|
447
|
+
finalDiff: 'diff --git a/scenario.yaml b/scenario.yaml',
|
|
448
|
+
trace: { changedPaths: ['scenario.yaml'] },
|
|
449
|
+
});
|
|
450
|
+
expect(outcome.evidence.events).toHaveLength(2);
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
it('uses the current local TrueForge URL by default', async () => {
|
|
454
|
+
let requestedUrl = '';
|
|
455
|
+
const adapter = new TrueForgeAdapter({
|
|
456
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
457
|
+
prompt: 'Repair the supplied scenario.',
|
|
458
|
+
collectOutcome: async () => ({ acceptancePassed: false, changedPaths: [] }),
|
|
459
|
+
fetch: async (input) => {
|
|
460
|
+
requestedUrl = String(input);
|
|
461
|
+
throw new Error('offline fixture');
|
|
462
|
+
},
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
await adapter.run(TASK);
|
|
466
|
+
|
|
467
|
+
expect(requestedUrl).toBe('http://localhost:8790/api/v1/sessions');
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
it('redacts configured secrets from all collected outcome strings', async () => {
|
|
471
|
+
const { client } = createClient();
|
|
472
|
+
const adapter = new TrueForgeAdapter(
|
|
473
|
+
{
|
|
474
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
475
|
+
prompt: 'Repair the supplied scenario.',
|
|
476
|
+
token: 'configured-exact-secret',
|
|
477
|
+
collectOutcome: async () => ({
|
|
478
|
+
acceptancePassed: true,
|
|
479
|
+
changedPaths: ['reports/configured-exact-secret.txt'],
|
|
480
|
+
finalDiff: 'diff containing configured-exact-secret',
|
|
481
|
+
}),
|
|
482
|
+
},
|
|
483
|
+
client
|
|
484
|
+
);
|
|
485
|
+
|
|
486
|
+
const outcome = await adapter.run(TASK);
|
|
487
|
+
|
|
488
|
+
expect(outcome.trace.changedPaths).toEqual(['reports/[REDACTED].txt']);
|
|
489
|
+
expect(outcome.finalDiff).toBe('diff containing [REDACTED]');
|
|
490
|
+
expect(JSON.stringify(outcome)).not.toContain('configured-exact-secret');
|
|
491
|
+
});
|
|
492
|
+
|
|
493
|
+
it('upserts provider and MCP settings only through explicit setup', async () => {
|
|
494
|
+
const { client, calls } = createClient();
|
|
495
|
+
const adapter = new TrueForgeAdapter(
|
|
496
|
+
{
|
|
497
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
498
|
+
prompt: 'Repair the supplied scenario.',
|
|
499
|
+
collectOutcome: async () => ({ acceptancePassed: false, changedPaths: [] }),
|
|
500
|
+
setup: {
|
|
501
|
+
provider: {
|
|
502
|
+
type: 'custom',
|
|
503
|
+
name: 'ling',
|
|
504
|
+
baseUrl: 'https://api.ant-ling.com/v1',
|
|
505
|
+
models: [{ modelId: 'Ling-3.0-flash', name: 'ling-3-flash', properties: {} }],
|
|
506
|
+
},
|
|
507
|
+
mcpServer: {
|
|
508
|
+
type: 'remote',
|
|
509
|
+
name: 'sandbox',
|
|
510
|
+
url: 'http://127.0.0.1:8787/mcp',
|
|
511
|
+
description: 'Local evaluation sandbox',
|
|
512
|
+
},
|
|
513
|
+
},
|
|
514
|
+
},
|
|
515
|
+
client
|
|
516
|
+
);
|
|
517
|
+
|
|
518
|
+
const result = await adapter.setup();
|
|
519
|
+
|
|
520
|
+
expect(calls.providers).toHaveLength(1);
|
|
521
|
+
expect(calls.mcpServers).toHaveLength(1);
|
|
522
|
+
expect(result).toEqual({ providerName: 'ling', mcpServerName: 'sandbox' });
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
it('does not treat pending required actions as a completed outcome', async () => {
|
|
526
|
+
const pending = {
|
|
527
|
+
...TURN_DONE,
|
|
528
|
+
state: {
|
|
529
|
+
...TURN_DONE.state,
|
|
530
|
+
requiredActions: [
|
|
531
|
+
{
|
|
532
|
+
type: 'tool.approval_required' as const,
|
|
533
|
+
id: 'approval-1',
|
|
534
|
+
createdAt: '2026-08-21T10:00:01.000Z',
|
|
535
|
+
threadId: 'main',
|
|
536
|
+
toolCalls: [{ id: 'call-1', sourceEventId: 'event-2' }],
|
|
537
|
+
},
|
|
538
|
+
],
|
|
539
|
+
},
|
|
540
|
+
};
|
|
541
|
+
const { client } = createClient([TURN_CREATED, pending]);
|
|
542
|
+
let collected = false;
|
|
543
|
+
const adapter = new TrueForgeAdapter(
|
|
544
|
+
{
|
|
545
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
546
|
+
prompt: 'Repair the supplied scenario.',
|
|
547
|
+
collectOutcome: async () => {
|
|
548
|
+
collected = true;
|
|
549
|
+
return {
|
|
550
|
+
acceptancePassed: true,
|
|
551
|
+
changedPaths: ['pending-change.yaml'],
|
|
552
|
+
finalDiff: 'pending diff',
|
|
553
|
+
};
|
|
554
|
+
},
|
|
555
|
+
},
|
|
556
|
+
client
|
|
557
|
+
);
|
|
558
|
+
|
|
559
|
+
const outcome = await adapter.run(TASK);
|
|
560
|
+
|
|
561
|
+
expect(collected).toBe(true);
|
|
562
|
+
expect(outcome.completed).toBe(false);
|
|
563
|
+
expect(outcome.acceptancePassed).toBe(true);
|
|
564
|
+
expect(outcome.trace.changedPaths).toEqual(['pending-change.yaml']);
|
|
565
|
+
expect(outcome.finalDiff).toBe('pending diff');
|
|
566
|
+
expect(outcome.error).toContain('requires additional action');
|
|
567
|
+
});
|
|
568
|
+
|
|
569
|
+
it('aborts timed-out turns and best-effort cancels their session', async () => {
|
|
570
|
+
const { client, calls } = createClient();
|
|
571
|
+
client.sessions.createTurnStream = async (_sessionId, _request, options) =>
|
|
572
|
+
(async function* () {
|
|
573
|
+
yield TURN_CREATED;
|
|
574
|
+
await new Promise<void>((_resolve, reject) => {
|
|
575
|
+
options?.abortSignal?.addEventListener('abort', () => reject(new Error('aborted')), {
|
|
576
|
+
once: true,
|
|
577
|
+
});
|
|
578
|
+
});
|
|
579
|
+
})();
|
|
580
|
+
const adapter = new TrueForgeAdapter(
|
|
581
|
+
{
|
|
582
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
583
|
+
prompt: 'Repair the supplied scenario.',
|
|
584
|
+
collectOutcome: async ({ sessionId, terminalState }) => {
|
|
585
|
+
expect(sessionId).toBe('session-1');
|
|
586
|
+
expect(terminalState).toBeUndefined();
|
|
587
|
+
return {
|
|
588
|
+
acceptancePassed: true,
|
|
589
|
+
changedPaths: ['timeout-change.yaml'],
|
|
590
|
+
finalDiff: 'timeout diff',
|
|
591
|
+
};
|
|
592
|
+
},
|
|
593
|
+
turnTimeoutMs: 5,
|
|
594
|
+
},
|
|
595
|
+
client
|
|
596
|
+
);
|
|
597
|
+
|
|
598
|
+
const outcome = await adapter.run(TASK);
|
|
599
|
+
|
|
600
|
+
expect(outcome.completed).toBe(false);
|
|
601
|
+
expect(outcome.acceptancePassed).toBe(true);
|
|
602
|
+
expect(outcome.error).toBe('TrueForge turn timed out after 5ms');
|
|
603
|
+
expect(outcome.trace.changedPaths).toEqual(['timeout-change.yaml']);
|
|
604
|
+
expect(outcome.finalDiff).toBe('timeout diff');
|
|
605
|
+
expect(calls.cancellations).toEqual(['session-1']);
|
|
606
|
+
});
|
|
607
|
+
|
|
608
|
+
it('applies the task deadline to outcome collection and bounds its failure grace', async () => {
|
|
609
|
+
const { client, calls } = createClient();
|
|
610
|
+
let collections = 0;
|
|
611
|
+
const adapter = new TrueForgeAdapter(
|
|
612
|
+
{
|
|
613
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
614
|
+
prompt: 'Repair the supplied scenario.',
|
|
615
|
+
collectOutcome: async () => {
|
|
616
|
+
collections += 1;
|
|
617
|
+
return new Promise(() => {});
|
|
618
|
+
},
|
|
619
|
+
failureCollectionGraceMs: 5,
|
|
620
|
+
turnTimeoutMs: 5,
|
|
621
|
+
},
|
|
622
|
+
client
|
|
623
|
+
);
|
|
624
|
+
|
|
625
|
+
const outcome = await Promise.race([
|
|
626
|
+
adapter.run(TASK),
|
|
627
|
+
new Promise<never>((_resolve, reject) =>
|
|
628
|
+
setTimeout(() => reject(new Error('collector was not bounded')), 100)
|
|
629
|
+
),
|
|
630
|
+
]);
|
|
631
|
+
|
|
632
|
+
expect(outcome.completed).toBe(false);
|
|
633
|
+
expect(outcome.error).toBe('TrueForge turn timed out after 5ms');
|
|
634
|
+
expect(collections).toBe(1);
|
|
635
|
+
expect(calls.cancellations).toEqual(['session-1']);
|
|
636
|
+
});
|
|
637
|
+
|
|
638
|
+
it('cancels a session that finishes creation after the task deadline', async () => {
|
|
639
|
+
const { client, calls } = createClient();
|
|
640
|
+
client.sessions.create = async () => {
|
|
641
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
642
|
+
return { data: { id: 'session-1' } };
|
|
643
|
+
};
|
|
644
|
+
const adapter = new TrueForgeAdapter(
|
|
645
|
+
{
|
|
646
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
647
|
+
prompt: 'Repair the supplied scenario.',
|
|
648
|
+
collectOutcome: async () => ({ acceptancePassed: false, changedPaths: [] }),
|
|
649
|
+
failureCollectionGraceMs: 5,
|
|
650
|
+
turnTimeoutMs: 5,
|
|
651
|
+
},
|
|
652
|
+
client
|
|
653
|
+
);
|
|
654
|
+
|
|
655
|
+
const outcome = await adapter.run(TASK);
|
|
656
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
657
|
+
|
|
658
|
+
expect(outcome.completed).toBe(false);
|
|
659
|
+
expect(outcome.error).toBe('TrueForge turn timed out after 5ms');
|
|
660
|
+
expect(calls.cancellations).toEqual(['session-1']);
|
|
661
|
+
expect(calls.turns).toEqual([]);
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
it('does not let best-effort cancellation hang a timed-out outcome', async () => {
|
|
665
|
+
const { client } = createClient();
|
|
666
|
+
client.sessions.createTurnStream = async (_sessionId, _request, options) =>
|
|
667
|
+
(async function* () {
|
|
668
|
+
yield TURN_CREATED;
|
|
669
|
+
await new Promise<void>((_resolve, reject) => {
|
|
670
|
+
options?.abortSignal?.addEventListener('abort', () => reject(new Error('aborted')), {
|
|
671
|
+
once: true,
|
|
672
|
+
});
|
|
673
|
+
});
|
|
674
|
+
})();
|
|
675
|
+
client.sessions.cancel = async () => new Promise(() => {});
|
|
676
|
+
const adapter = new TrueForgeAdapter(
|
|
677
|
+
{
|
|
678
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
679
|
+
prompt: 'Repair the supplied scenario.',
|
|
680
|
+
collectOutcome: async () => ({ acceptancePassed: false, changedPaths: [] }),
|
|
681
|
+
failureCollectionGraceMs: 5,
|
|
682
|
+
turnTimeoutMs: 5,
|
|
683
|
+
},
|
|
684
|
+
client
|
|
685
|
+
);
|
|
686
|
+
|
|
687
|
+
const outcome = await Promise.race([
|
|
688
|
+
adapter.run(TASK),
|
|
689
|
+
new Promise<never>((_resolve, reject) =>
|
|
690
|
+
setTimeout(() => reject(new Error('cancellation was not bounded')), 300)
|
|
691
|
+
),
|
|
692
|
+
]);
|
|
693
|
+
|
|
694
|
+
expect(outcome.completed).toBe(false);
|
|
695
|
+
expect(outcome.error).toBe('TrueForge turn timed out after 5ms');
|
|
696
|
+
});
|
|
697
|
+
|
|
698
|
+
it('cancels a non-terminal stream failure and retains collected failure evidence', async () => {
|
|
699
|
+
const { client, calls } = createClient();
|
|
700
|
+
client.sessions.createTurnStream = async () =>
|
|
701
|
+
(async function* () {
|
|
702
|
+
yield TURN_CREATED;
|
|
703
|
+
throw new Error('stream disconnected');
|
|
704
|
+
})();
|
|
705
|
+
const adapter = new TrueForgeAdapter(
|
|
706
|
+
{
|
|
707
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
708
|
+
prompt: 'Repair the supplied scenario.',
|
|
709
|
+
collectOutcome: async ({ sessionId, turnId, terminalState }) => {
|
|
710
|
+
expect(sessionId).toBe('session-1');
|
|
711
|
+
expect(turnId).toBe('turn-1');
|
|
712
|
+
expect(terminalState).toBeUndefined();
|
|
713
|
+
return {
|
|
714
|
+
acceptancePassed: false,
|
|
715
|
+
changedPaths: ['stream-change.yaml'],
|
|
716
|
+
finalDiff: 'stream diff',
|
|
717
|
+
};
|
|
718
|
+
},
|
|
719
|
+
},
|
|
720
|
+
client
|
|
721
|
+
);
|
|
722
|
+
|
|
723
|
+
const outcome = await adapter.run(TASK);
|
|
724
|
+
|
|
725
|
+
expect(outcome.completed).toBe(false);
|
|
726
|
+
expect(outcome.trace.changedPaths).toEqual(['stream-change.yaml']);
|
|
727
|
+
expect(outcome.finalDiff).toBe('stream diff');
|
|
728
|
+
expect(calls.cancellations).toEqual(['session-1']);
|
|
729
|
+
});
|
|
730
|
+
|
|
731
|
+
it('retains collected evidence after a terminal TrueForge error', async () => {
|
|
732
|
+
const terminalError = {
|
|
733
|
+
type: 'turn.done' as const,
|
|
734
|
+
id: 'event-error',
|
|
735
|
+
threadId: null,
|
|
736
|
+
createdAt: '2026-08-21T10:00:01.000Z',
|
|
737
|
+
state: {
|
|
738
|
+
status: 'error' as const,
|
|
739
|
+
message: 'agent failed',
|
|
740
|
+
completedAt: '2026-08-21T10:00:01.000Z',
|
|
741
|
+
},
|
|
742
|
+
};
|
|
743
|
+
const { client } = createClient([TURN_CREATED, terminalError]);
|
|
744
|
+
const adapter = new TrueForgeAdapter(
|
|
745
|
+
{
|
|
746
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
747
|
+
prompt: 'Repair the supplied scenario.',
|
|
748
|
+
collectOutcome: async ({ terminalState }) => {
|
|
749
|
+
expect(terminalState?.status).toBe('error');
|
|
750
|
+
return {
|
|
751
|
+
acceptancePassed: false,
|
|
752
|
+
changedPaths: ['terminal-error-change.yaml'],
|
|
753
|
+
finalDiff: 'terminal error diff',
|
|
754
|
+
};
|
|
755
|
+
},
|
|
756
|
+
},
|
|
757
|
+
client
|
|
758
|
+
);
|
|
759
|
+
|
|
760
|
+
const outcome = await adapter.run(TASK);
|
|
761
|
+
|
|
762
|
+
expect(outcome.completed).toBe(false);
|
|
763
|
+
expect(outcome.trace.changedPaths).toEqual(['terminal-error-change.yaml']);
|
|
764
|
+
expect(outcome.finalDiff).toBe('terminal error diff');
|
|
765
|
+
expect(outcome.error).toBe('agent failed');
|
|
766
|
+
});
|
|
767
|
+
|
|
768
|
+
it('returns a redacted structured outcome for routine API failures', async () => {
|
|
769
|
+
const { client } = createClient();
|
|
770
|
+
client.sessions.create = async () => {
|
|
771
|
+
throw new Error('request failed: Authorization: Bearer not-a-real-token');
|
|
772
|
+
};
|
|
773
|
+
const adapter = new TrueForgeAdapter(
|
|
774
|
+
{
|
|
775
|
+
agent: { model: { name: 'ling/ling-3-flash' } },
|
|
776
|
+
prompt: 'Repair the supplied scenario.',
|
|
777
|
+
token: 'not-a-real-token',
|
|
778
|
+
collectOutcome: async () => ({ acceptancePassed: false, changedPaths: [] }),
|
|
779
|
+
},
|
|
780
|
+
client
|
|
781
|
+
);
|
|
782
|
+
|
|
783
|
+
const outcome = await adapter.run(TASK);
|
|
784
|
+
|
|
785
|
+
expect(outcome.completed).toBe(false);
|
|
786
|
+
expect(outcome.acceptancePassed).toBe(false);
|
|
787
|
+
expect(outcome.error).toContain('[REDACTED]');
|
|
788
|
+
expect(JSON.stringify(outcome)).not.toContain('not-a-real-token');
|
|
789
|
+
});
|
|
790
|
+
});
|