@agent-native/core 0.173.1 → 0.173.2-nightly-20260825163448
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/templates/clips/desktop/src/app.tsx +77 -7
- package/corpus/templates/clips/desktop/src/lib/recorder.ts +91 -25
- package/corpus/templates/clips/desktop/src/lib/upload-recovery.ts +20 -2
- package/corpus/templates/clips/desktop/src-tauri/Cargo.toml +7 -1
- package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +1 -0
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +372 -70
- package/corpus/templates/clips/server/lib/resumable-session.ts +15 -0
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/chunk.post.ts +245 -118
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/reset-chunks.post.ts +21 -11
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/resume.get.ts +132 -25
- package/corpus/templates/content/actions/_database-property-input.ts +211 -0
- package/corpus/templates/content/actions/_database-row-mutation.ts +151 -26
- package/corpus/templates/content/actions/add-database-item.ts +15 -5
- package/corpus/templates/content/actions/update-database-item.ts +15 -7
- package/corpus/templates/content/actions/upsert-database-item-by-key.ts +15 -5
- package/corpus/templates/content/package.json +1 -0
- package/corpus/templates/content/parity/README.md +11 -2
- package/corpus/templates/content/parity/eval-scenarios.ts +52 -0
- package/corpus/templates/content/parity/run-database-create-property-preservation.ts +140 -0
- package/corpus/templates/content/parity/scenario-to-eval.ts +390 -0
- package/dist/agent/run-store.js +0 -1
- package/dist/client/agent-engine-key.js +26 -4
- package/dist/client/settings/ProviderSetupForm.js +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/eval/agent-runner.js +23 -0
- package/dist/eval/types.d.ts +11 -0
- package/dist/observability/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/server/agent-engine-api-key-route.d.ts +7 -0
- package/dist/server/agent-engine-api-key-route.js +38 -0
- package/dist/server/realtime-token.d.ts +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { createAgentRunner, runEvals } from "@agent-native/core/eval";
|
|
2
|
+
|
|
3
|
+
import addDatabaseItem from "../actions/add-database-item.ts";
|
|
4
|
+
import getContentDatabase from "../actions/get-content-database.ts";
|
|
5
|
+
import listContentDatabases from "../actions/list-content-databases.ts";
|
|
6
|
+
import { parityEvalScenarios } from "./eval-scenarios.ts";
|
|
7
|
+
import { scenarioToEval } from "./scenario-to-eval.ts";
|
|
8
|
+
|
|
9
|
+
const scenario = parityEvalScenarios.find(
|
|
10
|
+
(candidate) => candidate.id === "database-create-property-preservation",
|
|
11
|
+
);
|
|
12
|
+
if (!scenario) {
|
|
13
|
+
throw new Error("Missing database create property preservation scenario.");
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const FIXTURE_SPACE_ID = "fixture_personal_space";
|
|
17
|
+
const FIXTURE_DATABASE_ID = "fixture_feedback_database";
|
|
18
|
+
const FIXTURE_DOCUMENT_ID = "fixture_feedback_document";
|
|
19
|
+
const FIXTURE_SCHEMA_REVISION = "fixture_schema_revision";
|
|
20
|
+
const FIXTURE_DATABASE_TITLE = "PR #3314 feedback";
|
|
21
|
+
|
|
22
|
+
const evalCase = scenarioToEval(scenario);
|
|
23
|
+
evalCase.scorers = evalCase.scorers.filter(
|
|
24
|
+
(scorer) =>
|
|
25
|
+
scorer.name === "expected_tools" ||
|
|
26
|
+
scorer.name === "expected_property_values",
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
const runner = await createAgentRunner({
|
|
30
|
+
actions: {
|
|
31
|
+
"list-content-databases": {
|
|
32
|
+
...listContentDatabases,
|
|
33
|
+
run: async () => ({
|
|
34
|
+
databases: [
|
|
35
|
+
{
|
|
36
|
+
databaseId: FIXTURE_DATABASE_ID,
|
|
37
|
+
documentId: FIXTURE_DOCUMENT_ID,
|
|
38
|
+
spaceId: FIXTURE_SPACE_ID,
|
|
39
|
+
title: FIXTURE_DATABASE_TITLE,
|
|
40
|
+
description: "",
|
|
41
|
+
},
|
|
42
|
+
],
|
|
43
|
+
pagination: {
|
|
44
|
+
offset: 0,
|
|
45
|
+
limit: 50,
|
|
46
|
+
totalItems: 1,
|
|
47
|
+
returnedItems: 1,
|
|
48
|
+
hasMore: false,
|
|
49
|
+
nextOffset: null,
|
|
50
|
+
},
|
|
51
|
+
}),
|
|
52
|
+
},
|
|
53
|
+
"get-content-database": {
|
|
54
|
+
...getContentDatabase,
|
|
55
|
+
run: async () => ({
|
|
56
|
+
database: {
|
|
57
|
+
id: FIXTURE_DATABASE_ID,
|
|
58
|
+
documentId: FIXTURE_DOCUMENT_ID,
|
|
59
|
+
spaceId: FIXTURE_SPACE_ID,
|
|
60
|
+
title: FIXTURE_DATABASE_TITLE,
|
|
61
|
+
naturalKeyPropertyId: null,
|
|
62
|
+
viewConfig: {
|
|
63
|
+
activeViewId: "fixture_view",
|
|
64
|
+
views: [],
|
|
65
|
+
sorts: [],
|
|
66
|
+
filters: [],
|
|
67
|
+
columnWidths: {},
|
|
68
|
+
},
|
|
69
|
+
createdAt: new Date(0).toISOString(),
|
|
70
|
+
updatedAt: new Date(0).toISOString(),
|
|
71
|
+
},
|
|
72
|
+
properties: [],
|
|
73
|
+
items: [],
|
|
74
|
+
source: null,
|
|
75
|
+
mutationContract: {
|
|
76
|
+
target: {
|
|
77
|
+
authorityScope: {
|
|
78
|
+
kind: "personal",
|
|
79
|
+
id: "fixture-owner@example.com",
|
|
80
|
+
},
|
|
81
|
+
spaceId: FIXTURE_SPACE_ID,
|
|
82
|
+
databaseId: FIXTURE_DATABASE_ID,
|
|
83
|
+
databaseDocumentId: FIXTURE_DOCUMENT_ID,
|
|
84
|
+
},
|
|
85
|
+
schemaRevision: FIXTURE_SCHEMA_REVISION,
|
|
86
|
+
naturalKeyPropertyId: null,
|
|
87
|
+
properties: [
|
|
88
|
+
{
|
|
89
|
+
id: "fixture_status_property",
|
|
90
|
+
name: "Status",
|
|
91
|
+
type: "status",
|
|
92
|
+
writable: true,
|
|
93
|
+
sourceManaged: false,
|
|
94
|
+
acceptedShape: null,
|
|
95
|
+
options: {
|
|
96
|
+
options: [
|
|
97
|
+
{
|
|
98
|
+
id: "status-cannot-verify",
|
|
99
|
+
name: "Cannot verify",
|
|
100
|
+
color: "gray",
|
|
101
|
+
},
|
|
102
|
+
],
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
id: "fixture_evidence_property",
|
|
107
|
+
name: "Evidence",
|
|
108
|
+
type: "text",
|
|
109
|
+
writable: true,
|
|
110
|
+
sourceManaged: false,
|
|
111
|
+
acceptedShape: null,
|
|
112
|
+
options: {},
|
|
113
|
+
},
|
|
114
|
+
],
|
|
115
|
+
},
|
|
116
|
+
}),
|
|
117
|
+
},
|
|
118
|
+
"add-database-item": {
|
|
119
|
+
...addDatabaseItem,
|
|
120
|
+
run: async (input) => ({ fixtureOnly: true, received: input }),
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
systemPrompt:
|
|
124
|
+
"You are Content's AI document assistant. Use the registered Content actions and preserve exact user-supplied target constraints, property IDs, and property values. Never invent fields or claim an action succeeded when it failed.",
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
const report = await runEvals([evalCase], runner, { persist: false });
|
|
128
|
+
console.log(
|
|
129
|
+
JSON.stringify(
|
|
130
|
+
{
|
|
131
|
+
engine: runner.engine.name,
|
|
132
|
+
model: runner.model,
|
|
133
|
+
report,
|
|
134
|
+
},
|
|
135
|
+
null,
|
|
136
|
+
2,
|
|
137
|
+
),
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
process.exitCode = report.failed === 0 ? 0 : 1;
|
|
@@ -30,6 +30,387 @@ function expectedToolScorer(expectedTools: string[]) {
|
|
|
30
30
|
});
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
function hasExactKeys(
|
|
34
|
+
record: Record<string, unknown>,
|
|
35
|
+
expectedKeys: readonly string[],
|
|
36
|
+
): boolean {
|
|
37
|
+
const actualKeys = Object.keys(record).sort();
|
|
38
|
+
const sortedExpectedKeys = [...expectedKeys].sort();
|
|
39
|
+
return (
|
|
40
|
+
actualKeys.length === sortedExpectedKeys.length &&
|
|
41
|
+
actualKeys.every((key, index) => key === sortedExpectedKeys[index])
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function analyzePropertyValues(input: unknown): {
|
|
46
|
+
received: Record<string, unknown>;
|
|
47
|
+
receivedTypes: Record<string, string>;
|
|
48
|
+
invalid: string[];
|
|
49
|
+
} {
|
|
50
|
+
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
|
51
|
+
return {
|
|
52
|
+
received: {},
|
|
53
|
+
receivedTypes: {},
|
|
54
|
+
invalid: ["tool input is not an object"],
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
const record = input as Record<string, unknown>;
|
|
58
|
+
const hasEntries = record.propertyEntries !== undefined;
|
|
59
|
+
const hasValues = record.propertyValues !== undefined;
|
|
60
|
+
if (hasEntries && hasValues) {
|
|
61
|
+
return {
|
|
62
|
+
received: {},
|
|
63
|
+
receivedTypes: {},
|
|
64
|
+
invalid: ["propertyEntries and propertyValues were both provided"],
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
if (hasValues) {
|
|
68
|
+
if (
|
|
69
|
+
!record.propertyValues ||
|
|
70
|
+
typeof record.propertyValues !== "object" ||
|
|
71
|
+
Array.isArray(record.propertyValues)
|
|
72
|
+
) {
|
|
73
|
+
return {
|
|
74
|
+
received: {},
|
|
75
|
+
receivedTypes: {},
|
|
76
|
+
invalid: ["propertyValues is not a record"],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
received: record.propertyValues as Record<string, unknown>,
|
|
81
|
+
receivedTypes: {},
|
|
82
|
+
invalid: ["propertyValues bypassed the typed agent input"],
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
if (!hasEntries || !Array.isArray(record.propertyEntries)) {
|
|
86
|
+
return {
|
|
87
|
+
received: {},
|
|
88
|
+
receivedTypes: {},
|
|
89
|
+
invalid: ["propertyEntries is not an array"],
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const received: Record<string, unknown> = Object.create(null) as Record<
|
|
94
|
+
string,
|
|
95
|
+
unknown
|
|
96
|
+
>;
|
|
97
|
+
const receivedTypes: Record<string, string> = Object.create(null) as Record<
|
|
98
|
+
string,
|
|
99
|
+
string
|
|
100
|
+
>;
|
|
101
|
+
const invalid: string[] = [];
|
|
102
|
+
for (const entry of record.propertyEntries) {
|
|
103
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
|
104
|
+
invalid.push("propertyEntries contains a non-object entry");
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
const { propertyId, propertyType, value } = entry as Record<
|
|
108
|
+
string,
|
|
109
|
+
unknown
|
|
110
|
+
>;
|
|
111
|
+
if (
|
|
112
|
+
!hasExactKeys(entry as Record<string, unknown>, [
|
|
113
|
+
"propertyId",
|
|
114
|
+
"propertyType",
|
|
115
|
+
"value",
|
|
116
|
+
])
|
|
117
|
+
) {
|
|
118
|
+
invalid.push(
|
|
119
|
+
"propertyEntries contains an entry with unrecognized fields",
|
|
120
|
+
);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
if (typeof propertyId !== "string" || propertyId.length === 0) {
|
|
124
|
+
invalid.push("propertyEntries contains an invalid propertyId");
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (typeof propertyType !== "string" || propertyType.length === 0) {
|
|
128
|
+
invalid.push(
|
|
129
|
+
`propertyEntries contains an invalid type for ${propertyId}`,
|
|
130
|
+
);
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
if (Object.prototype.hasOwnProperty.call(received, propertyId)) {
|
|
134
|
+
invalid.push(`propertyEntries contains duplicate ID ${propertyId}`);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
received[propertyId] = value;
|
|
138
|
+
receivedTypes[propertyId] = propertyType;
|
|
139
|
+
}
|
|
140
|
+
return { received, receivedTypes, invalid };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const databaseRowMutationTools = new Set([
|
|
144
|
+
"add-database-item",
|
|
145
|
+
"update-database-item",
|
|
146
|
+
"upsert-database-item-by-key",
|
|
147
|
+
"duplicate-database-items",
|
|
148
|
+
"remove-database-items",
|
|
149
|
+
]);
|
|
150
|
+
|
|
151
|
+
function matchesCreateEnvelope(
|
|
152
|
+
input: Record<string, unknown>,
|
|
153
|
+
expected: NonNullable<ParityEvalScenario["expectedCreateEnvelope"]>,
|
|
154
|
+
): boolean {
|
|
155
|
+
const target = input.target;
|
|
156
|
+
if (!target || typeof target !== "object" || Array.isArray(target)) {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
const actualTarget = target as Record<string, unknown>;
|
|
160
|
+
return (
|
|
161
|
+
hasExactKeys(input, [
|
|
162
|
+
"target",
|
|
163
|
+
"expectedSchemaRevision",
|
|
164
|
+
"idempotencyKey",
|
|
165
|
+
"title",
|
|
166
|
+
input.propertyEntries === undefined
|
|
167
|
+
? "propertyValues"
|
|
168
|
+
: "propertyEntries",
|
|
169
|
+
]) &&
|
|
170
|
+
hasExactKeys(actualTarget, [
|
|
171
|
+
"spaceId",
|
|
172
|
+
"databaseId",
|
|
173
|
+
"databaseDocumentId",
|
|
174
|
+
]) &&
|
|
175
|
+
actualTarget.spaceId === expected.target.spaceId &&
|
|
176
|
+
actualTarget.databaseId === expected.target.databaseId &&
|
|
177
|
+
actualTarget.databaseDocumentId === expected.target.databaseDocumentId &&
|
|
178
|
+
input.expectedSchemaRevision === expected.expectedSchemaRevision &&
|
|
179
|
+
input.idempotencyKey === expected.idempotencyKey &&
|
|
180
|
+
input.title === expected.title
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function parseToolResultJson(result: string | undefined): unknown {
|
|
185
|
+
if (!result) return undefined;
|
|
186
|
+
try {
|
|
187
|
+
return JSON.parse(result);
|
|
188
|
+
} catch {
|
|
189
|
+
// coercion-ok: an unparseable tool result fails the same "did not supply
|
|
190
|
+
// the expected target/contract" scorer checks below as a missing one —
|
|
191
|
+
// both are non-passing evidence, not a distinction the eval needs to make.
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function expectedPropertyValuesScorer(
|
|
197
|
+
expected: Record<string, unknown>,
|
|
198
|
+
expectedTypes: Record<string, string> | undefined,
|
|
199
|
+
expectedEnvelope?: ParityEvalScenario["expectedCreateEnvelope"],
|
|
200
|
+
) {
|
|
201
|
+
return createScorer<
|
|
202
|
+
AgentRunOutput,
|
|
203
|
+
{
|
|
204
|
+
received: Record<string, unknown>;
|
|
205
|
+
missing: string[];
|
|
206
|
+
unexpected: string[];
|
|
207
|
+
invalid: string[];
|
|
208
|
+
mutationCalls: string[];
|
|
209
|
+
}
|
|
210
|
+
>({
|
|
211
|
+
name: "expected_property_values",
|
|
212
|
+
analyze(run) {
|
|
213
|
+
const mutationCalls = (run.toolCallDetails ?? [])
|
|
214
|
+
.filter((call) => databaseRowMutationTools.has(call.name))
|
|
215
|
+
.map((call) => call.name);
|
|
216
|
+
const createCalls = (run.toolCallDetails ?? []).filter(
|
|
217
|
+
(call) => call.name === "add-database-item",
|
|
218
|
+
);
|
|
219
|
+
const analysis = analyzePropertyValues(createCalls[0]?.input);
|
|
220
|
+
const invalid = [...analysis.invalid];
|
|
221
|
+
if (createCalls.length !== 1) {
|
|
222
|
+
invalid.push(
|
|
223
|
+
`expected exactly one add-database-item call, received ${createCalls.length}`,
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
if (mutationCalls.length !== 1) {
|
|
227
|
+
invalid.push(
|
|
228
|
+
`expected exactly one row mutation, received ${mutationCalls.length}`,
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
const orderedCalls = run.toolCalls;
|
|
232
|
+
const listIndex = orderedCalls.indexOf("list-content-databases");
|
|
233
|
+
const inspectIndex = orderedCalls.indexOf("get-content-database");
|
|
234
|
+
const createIndex = orderedCalls.indexOf("add-database-item");
|
|
235
|
+
if (
|
|
236
|
+
listIndex < 0 ||
|
|
237
|
+
inspectIndex <= listIndex ||
|
|
238
|
+
createIndex <= inspectIndex
|
|
239
|
+
) {
|
|
240
|
+
invalid.push("database discovery did not precede the create mutation");
|
|
241
|
+
}
|
|
242
|
+
const listCall = (run.toolCallDetails ?? []).find(
|
|
243
|
+
(call) => call.name === "list-content-databases",
|
|
244
|
+
);
|
|
245
|
+
const inspectCall = (run.toolCallDetails ?? []).find(
|
|
246
|
+
(call) => call.name === "get-content-database",
|
|
247
|
+
);
|
|
248
|
+
const createCall = createCalls[0];
|
|
249
|
+
const listInput = listCall?.input as Record<string, unknown> | undefined;
|
|
250
|
+
const inspectInput = inspectCall?.input as
|
|
251
|
+
| Record<string, unknown>
|
|
252
|
+
| undefined;
|
|
253
|
+
if (
|
|
254
|
+
expectedEnvelope &&
|
|
255
|
+
(listInput?.title !== "PR #3314 feedback" ||
|
|
256
|
+
inspectInput?.databaseId !== expectedEnvelope.target.databaseId)
|
|
257
|
+
) {
|
|
258
|
+
invalid.push(
|
|
259
|
+
"discovery did not resolve the requested title to the exact create target",
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
if (
|
|
263
|
+
!listCall?.completed ||
|
|
264
|
+
listCall.isError ||
|
|
265
|
+
!inspectCall?.completed ||
|
|
266
|
+
inspectCall.isError
|
|
267
|
+
) {
|
|
268
|
+
invalid.push("database discovery calls did not complete successfully");
|
|
269
|
+
}
|
|
270
|
+
if (
|
|
271
|
+
listCall?.completedAtEventIndex === undefined ||
|
|
272
|
+
inspectCall?.startedAtEventIndex === undefined ||
|
|
273
|
+
listCall.completedAtEventIndex >= inspectCall.startedAtEventIndex ||
|
|
274
|
+
inspectCall.completedAtEventIndex === undefined ||
|
|
275
|
+
createCall?.startedAtEventIndex === undefined ||
|
|
276
|
+
inspectCall.completedAtEventIndex >= createCall.startedAtEventIndex
|
|
277
|
+
) {
|
|
278
|
+
invalid.push(
|
|
279
|
+
"database discovery results were not available before dependent calls started",
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
if (expectedEnvelope) {
|
|
283
|
+
const listResult = parseToolResultJson(listCall?.result) as
|
|
284
|
+
| { databases?: Array<Record<string, unknown>> }
|
|
285
|
+
| undefined;
|
|
286
|
+
const discoveredDatabase = listResult?.databases?.find(
|
|
287
|
+
(database) =>
|
|
288
|
+
database.databaseId === expectedEnvelope.target.databaseId,
|
|
289
|
+
);
|
|
290
|
+
const discoveredDocumentId =
|
|
291
|
+
discoveredDatabase?.documentId ??
|
|
292
|
+
discoveredDatabase?.databaseDocumentId;
|
|
293
|
+
if (
|
|
294
|
+
!discoveredDatabase ||
|
|
295
|
+
discoveredDocumentId !== expectedEnvelope.target.databaseDocumentId ||
|
|
296
|
+
discoveredDatabase.spaceId !== expectedEnvelope.target.spaceId
|
|
297
|
+
) {
|
|
298
|
+
invalid.push(
|
|
299
|
+
"list-content-databases result did not supply the expected create target",
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
const inspectResult = parseToolResultJson(inspectCall?.result) as
|
|
303
|
+
| {
|
|
304
|
+
mutationContract?: {
|
|
305
|
+
target?: Record<string, unknown>;
|
|
306
|
+
schemaRevision?: string;
|
|
307
|
+
expectedSchemaRevision?: string;
|
|
308
|
+
properties?: Array<Record<string, unknown>>;
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
| undefined;
|
|
312
|
+
const contractTarget = inspectResult?.mutationContract?.target;
|
|
313
|
+
const contractSchemaRevision =
|
|
314
|
+
inspectResult?.mutationContract?.schemaRevision ??
|
|
315
|
+
inspectResult?.mutationContract?.expectedSchemaRevision;
|
|
316
|
+
if (
|
|
317
|
+
!contractTarget ||
|
|
318
|
+
contractTarget.spaceId !== expectedEnvelope.target.spaceId ||
|
|
319
|
+
contractTarget.databaseId !== expectedEnvelope.target.databaseId ||
|
|
320
|
+
contractTarget.databaseDocumentId !==
|
|
321
|
+
expectedEnvelope.target.databaseDocumentId ||
|
|
322
|
+
contractSchemaRevision !== expectedEnvelope.expectedSchemaRevision
|
|
323
|
+
) {
|
|
324
|
+
invalid.push(
|
|
325
|
+
"get-content-database result did not supply the expected mutation contract",
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
const discoveredProperties = new Map(
|
|
329
|
+
(inspectResult?.mutationContract?.properties ?? []).map(
|
|
330
|
+
(property) => [property.id, property],
|
|
331
|
+
),
|
|
332
|
+
);
|
|
333
|
+
for (const [propertyId, expectedType] of Object.entries(
|
|
334
|
+
expectedTypes ?? {},
|
|
335
|
+
)) {
|
|
336
|
+
const property = discoveredProperties.get(propertyId);
|
|
337
|
+
if (
|
|
338
|
+
property?.type !== expectedType ||
|
|
339
|
+
property.writable !== true ||
|
|
340
|
+
property.sourceManaged === true
|
|
341
|
+
) {
|
|
342
|
+
invalid.push(
|
|
343
|
+
`discovery did not supply writable property ${propertyId} with type ${expectedType}`,
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
for (const [propertyId, expectedType] of Object.entries(
|
|
349
|
+
expectedTypes ?? {},
|
|
350
|
+
)) {
|
|
351
|
+
if (analysis.receivedTypes[propertyId] !== expectedType) {
|
|
352
|
+
invalid.push(
|
|
353
|
+
`property ${propertyId} did not declare discovered type ${expectedType}`,
|
|
354
|
+
);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
const createInput = createCalls[0]?.input;
|
|
358
|
+
if (
|
|
359
|
+
expectedEnvelope &&
|
|
360
|
+
(!createInput ||
|
|
361
|
+
typeof createInput !== "object" ||
|
|
362
|
+
Array.isArray(createInput) ||
|
|
363
|
+
!matchesCreateEnvelope(
|
|
364
|
+
createInput as Record<string, unknown>,
|
|
365
|
+
expectedEnvelope,
|
|
366
|
+
))
|
|
367
|
+
) {
|
|
368
|
+
invalid.push(
|
|
369
|
+
"create target, schema revision, idempotency key, or title did not match the fixture",
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
if (
|
|
373
|
+
!createCalls[0]?.completed ||
|
|
374
|
+
createCalls[0]?.completedSideEffect !== true ||
|
|
375
|
+
createCalls[0]?.isError
|
|
376
|
+
) {
|
|
377
|
+
invalid.push("add-database-item did not complete successfully");
|
|
378
|
+
}
|
|
379
|
+
if (!run.ok) {
|
|
380
|
+
invalid.push("agent run did not complete successfully");
|
|
381
|
+
}
|
|
382
|
+
const received = analysis.received;
|
|
383
|
+
const missing = Object.entries(expected)
|
|
384
|
+
.filter(([propertyId, value]) => received[propertyId] !== value)
|
|
385
|
+
.map(([propertyId]) => propertyId);
|
|
386
|
+
const unexpected = Object.keys(received).filter(
|
|
387
|
+
(propertyId) =>
|
|
388
|
+
!Object.prototype.hasOwnProperty.call(expected, propertyId),
|
|
389
|
+
);
|
|
390
|
+
return { received, missing, unexpected, invalid, mutationCalls };
|
|
391
|
+
},
|
|
392
|
+
generateScore({ missing, unexpected, invalid }) {
|
|
393
|
+
return missing.length === 0 &&
|
|
394
|
+
unexpected.length === 0 &&
|
|
395
|
+
invalid.length === 0
|
|
396
|
+
? 1
|
|
397
|
+
: 0;
|
|
398
|
+
},
|
|
399
|
+
generateReason({
|
|
400
|
+
analysis: { received, missing, unexpected, invalid, mutationCalls },
|
|
401
|
+
}) {
|
|
402
|
+
if (
|
|
403
|
+
missing.length === 0 &&
|
|
404
|
+
unexpected.length === 0 &&
|
|
405
|
+
invalid.length === 0
|
|
406
|
+
) {
|
|
407
|
+
return "Agent preserved every expected property ID and exact value without inventing another property.";
|
|
408
|
+
}
|
|
409
|
+
return `Received propertyValues ${JSON.stringify(received)}; mutations: ${mutationCalls.join(", ") || "none"}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}; invalid: ${invalid.join("; ") || "none"}`;
|
|
410
|
+
},
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
|
|
33
414
|
export function scenarioToEval(scenario: ParityEvalScenario): Eval {
|
|
34
415
|
const name = `content-parity:${scenario.id}`;
|
|
35
416
|
|
|
@@ -52,6 +433,15 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval {
|
|
|
52
433
|
...(scenario.expectedTools?.length
|
|
53
434
|
? [expectedToolScorer(scenario.expectedTools)]
|
|
54
435
|
: []),
|
|
436
|
+
...(scenario.expectedPropertyValues
|
|
437
|
+
? [
|
|
438
|
+
expectedPropertyValuesScorer(
|
|
439
|
+
scenario.expectedPropertyValues,
|
|
440
|
+
scenario.expectedPropertyTypes,
|
|
441
|
+
scenario.expectedCreateEnvelope,
|
|
442
|
+
),
|
|
443
|
+
]
|
|
444
|
+
: []),
|
|
55
445
|
],
|
|
56
446
|
});
|
|
57
447
|
}
|
package/dist/agent/run-store.js
CHANGED
|
@@ -2246,7 +2246,6 @@ export function resolveErroredRunTerminalEvent(run) {
|
|
|
2246
2246
|
type: "error",
|
|
2247
2247
|
error: detail || "The agent run failed.",
|
|
2248
2248
|
...(code && code !== "unknown" ? { errorCode: code } : {}),
|
|
2249
|
-
recoverable: true,
|
|
2250
2249
|
},
|
|
2251
2250
|
shouldPersist: true,
|
|
2252
2251
|
};
|
|
@@ -32,6 +32,31 @@ function dispatchConfiguredChanged() {
|
|
|
32
32
|
window.dispatchEvent(new CustomEvent(CONFIGURED_CHANGED_EVENT));
|
|
33
33
|
}
|
|
34
34
|
}
|
|
35
|
+
async function readProviderSettingsError(response) {
|
|
36
|
+
const text = await response.text();
|
|
37
|
+
const trimmed = text.trim();
|
|
38
|
+
if (!trimmed)
|
|
39
|
+
return undefined;
|
|
40
|
+
try {
|
|
41
|
+
const body = JSON.parse(trimmed);
|
|
42
|
+
if (body !== null && typeof body === "object") {
|
|
43
|
+
const error = body.error;
|
|
44
|
+
if (typeof error === "string" && error.trim()) {
|
|
45
|
+
return error.trim();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
else if (typeof body === "string" && body.trim()) {
|
|
49
|
+
return body.trim();
|
|
50
|
+
}
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
if (!(error instanceof SyntaxError))
|
|
55
|
+
throw error;
|
|
56
|
+
// Plain-text relay errors are the useful fallback for desktop requests.
|
|
57
|
+
}
|
|
58
|
+
return trimmed.startsWith("<") ? undefined : trimmed.slice(0, 500);
|
|
59
|
+
}
|
|
35
60
|
/**
|
|
36
61
|
* Persist a provider API key for the current owner. Resolves on success.
|
|
37
62
|
* Throws an Error with a readable message on failure. On success it also
|
|
@@ -68,10 +93,7 @@ export async function saveAgentEngineProviderSettings({ provider, key, apiKey, b
|
|
|
68
93
|
}),
|
|
69
94
|
});
|
|
70
95
|
if (!res.ok) {
|
|
71
|
-
const message = await res
|
|
72
|
-
.json()
|
|
73
|
-
.then((body) => body?.error)
|
|
74
|
-
.catch(() => null);
|
|
96
|
+
const message = await readProviderSettingsError(res);
|
|
75
97
|
throw new Error(message ??
|
|
76
98
|
(res.status === 401
|
|
77
99
|
? "Sign in to save a key, or connect Builder with a free tier instead."
|
|
@@ -141,5 +141,5 @@ export function AgentProviderSetupForm({ initialProvider = "anthropic", configur
|
|
|
141
141
|
})) : (t("agentPanel.saveAndUseProvider", {
|
|
142
142
|
provider: active.label,
|
|
143
143
|
defaultValue: `Save and use ${active.label}`,
|
|
144
|
-
})) }), active.docsUrl ? (_jsxs("a", { href: active.docsUrl, target: "_blank", rel: "noopener noreferrer", className: "inline-flex items-center gap-1 rounded-md px-1.5 py-1 text-[10px] font-medium text-muted-foreground no-underline transition-colors hover:bg-accent/40 hover:text-foreground", children: [t("agentPanel.getApiKey", { defaultValue: "Get an API key" }), _jsx(IconExternalLink, { size: 11 })] })) : null] }), error ? (
|
|
144
|
+
})) }), active.docsUrl ? (_jsxs("a", { href: active.docsUrl, target: "_blank", rel: "noopener noreferrer", className: "inline-flex items-center gap-1 rounded-md px-1.5 py-1 text-[10px] font-medium text-muted-foreground no-underline transition-colors hover:bg-accent/40 hover:text-foreground", children: [t("agentPanel.getApiKey", { defaultValue: "Get an API key" }), _jsx(IconExternalLink, { size: 11 })] })) : null] }), error ? (_jsxs("div", { role: "alert", "aria-live": "polite", className: "space-y-1 text-[11px] leading-relaxed text-destructive", children: [_jsx("p", { children: error }), active.docsUrl ? (_jsxs("a", { href: active.docsUrl, target: "_blank", rel: "noopener noreferrer", className: "inline-flex items-center gap-1 font-medium text-foreground underline underline-offset-2", children: [t("agentPanel.getApiKey", { defaultValue: "Get an API key" }), _jsx(IconExternalLink, { size: 11 })] })) : null] })) : null] })] }));
|
|
145
145
|
}
|
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
* Body: { json: any, fieldName?: string, type?: "map"|"array", requestSource?: string }
|
|
14
14
|
*/
|
|
15
15
|
export declare const postCollabJson: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
16
|
-
error: string;
|
|
17
16
|
ok?: undefined;
|
|
17
|
+
error: string;
|
|
18
18
|
} | {
|
|
19
19
|
error?: undefined;
|
|
20
20
|
ok: boolean;
|
|
@@ -52,19 +52,41 @@ export async function createAgentRunner(config) {
|
|
|
52
52
|
const messages = toEngineMessages(input);
|
|
53
53
|
let text = "";
|
|
54
54
|
const toolCalls = [];
|
|
55
|
+
const toolCallDetails = [];
|
|
55
56
|
let ok = true;
|
|
56
57
|
let error;
|
|
58
|
+
let eventIndex = 0;
|
|
57
59
|
const controller = new AbortController();
|
|
58
60
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
59
61
|
const started = Date.now();
|
|
60
62
|
const send = (event) => {
|
|
63
|
+
const currentEventIndex = eventIndex++;
|
|
61
64
|
switch (event.type) {
|
|
62
65
|
case "text":
|
|
63
66
|
text += event.text;
|
|
64
67
|
break;
|
|
65
68
|
case "tool_start":
|
|
66
69
|
toolCalls.push(event.tool);
|
|
70
|
+
toolCallDetails.push({
|
|
71
|
+
name: event.tool,
|
|
72
|
+
id: event.id,
|
|
73
|
+
input: event.input,
|
|
74
|
+
startedAtEventIndex: currentEventIndex,
|
|
75
|
+
});
|
|
76
|
+
break;
|
|
77
|
+
case "tool_done": {
|
|
78
|
+
const detail = event.id
|
|
79
|
+
? toolCallDetails.find((call) => call.id === event.id)
|
|
80
|
+
: toolCallDetails.find((call) => call.name === event.tool && !call.completed);
|
|
81
|
+
if (detail) {
|
|
82
|
+
detail.completed = true;
|
|
83
|
+
detail.completedAtEventIndex = currentEventIndex;
|
|
84
|
+
detail.completedSideEffect = event.completedSideEffect;
|
|
85
|
+
detail.isError = event.isError === true;
|
|
86
|
+
detail.result = event.result;
|
|
87
|
+
}
|
|
67
88
|
break;
|
|
89
|
+
}
|
|
68
90
|
case "error":
|
|
69
91
|
ok = false;
|
|
70
92
|
error = event.error;
|
|
@@ -95,6 +117,7 @@ export async function createAgentRunner(config) {
|
|
|
95
117
|
return {
|
|
96
118
|
text,
|
|
97
119
|
toolCalls,
|
|
120
|
+
toolCallDetails: toolCallDetails.map(({ id: _id, ...detail }) => detail),
|
|
98
121
|
ok,
|
|
99
122
|
error,
|
|
100
123
|
runId,
|
package/dist/eval/types.d.ts
CHANGED
|
@@ -27,6 +27,17 @@ export interface AgentRunOutput {
|
|
|
27
27
|
readonly text: string;
|
|
28
28
|
/** Names of tools/actions the agent invoked, in call order. */
|
|
29
29
|
readonly toolCalls: readonly string[];
|
|
30
|
+
/** Tool names, model-produced inputs, and execution outcomes in call order. */
|
|
31
|
+
readonly toolCallDetails?: readonly {
|
|
32
|
+
readonly name: string;
|
|
33
|
+
readonly input: unknown;
|
|
34
|
+
readonly startedAtEventIndex?: number;
|
|
35
|
+
readonly completedAtEventIndex?: number;
|
|
36
|
+
readonly completed?: boolean;
|
|
37
|
+
readonly completedSideEffect?: boolean;
|
|
38
|
+
readonly isError?: boolean;
|
|
39
|
+
readonly result?: string;
|
|
40
|
+
}[];
|
|
30
41
|
/** Whether the run completed without a terminal error event. */
|
|
31
42
|
readonly ok: boolean;
|
|
32
43
|
/** Terminal error message, if the run errored. */
|