@agentuity/runtime 0.0.100 → 0.0.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +34 -212
- package/dist/_metadata.d.ts +107 -0
- package/dist/_metadata.d.ts.map +1 -0
- package/dist/_metadata.js +179 -0
- package/dist/_metadata.js.map +1 -0
- package/dist/_process-protection.d.ts.map +1 -1
- package/dist/_process-protection.js +4 -0
- package/dist/_process-protection.js.map +1 -1
- package/dist/_services.d.ts.map +1 -1
- package/dist/_services.js +18 -17
- package/dist/_services.js.map +1 -1
- package/dist/_standalone.d.ts.map +1 -1
- package/dist/_standalone.js +17 -0
- package/dist/_standalone.js.map +1 -1
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +53 -12
- package/dist/agent.js.map +1 -1
- package/dist/app.d.ts +61 -10
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js.map +1 -1
- package/dist/devmode.d.ts.map +1 -1
- package/dist/devmode.js +13 -5
- package/dist/devmode.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/middleware.d.ts +61 -5
- package/dist/middleware.d.ts.map +1 -1
- package/dist/middleware.js +192 -25
- package/dist/middleware.js.map +1 -1
- package/dist/services/evalrun/http.d.ts.map +1 -1
- package/dist/services/evalrun/http.js +14 -4
- package/dist/services/evalrun/http.js.map +1 -1
- package/dist/services/local/vector.d.ts +5 -1
- package/dist/services/local/vector.d.ts.map +1 -1
- package/dist/services/local/vector.js +112 -0
- package/dist/services/local/vector.js.map +1 -1
- package/dist/services/session/http.d.ts.map +1 -1
- package/dist/services/session/http.js +7 -0
- package/dist/services/session/http.js.map +1 -1
- package/dist/services/session/local.d.ts +2 -2
- package/dist/services/session/local.d.ts.map +1 -1
- package/dist/services/session/local.js +5 -4
- package/dist/services/session/local.js.map +1 -1
- package/dist/session.d.ts +30 -4
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +90 -13
- package/dist/session.js.map +1 -1
- package/dist/workbench.d.ts.map +1 -1
- package/dist/workbench.js +13 -20
- package/dist/workbench.js.map +1 -1
- package/package.json +5 -5
- package/src/_metadata.ts +307 -0
- package/src/_process-protection.ts +6 -0
- package/src/_services.ts +23 -21
- package/src/_standalone.ts +22 -0
- package/src/agent.ts +63 -12
- package/src/app.ts +65 -9
- package/src/devmode.ts +16 -5
- package/src/index.ts +12 -2
- package/src/middleware.ts +221 -29
- package/src/services/evalrun/http.ts +15 -4
- package/src/services/local/vector.ts +160 -0
- package/src/services/session/http.ts +11 -0
- package/src/services/session/local.ts +9 -4
- package/src/session.ts +142 -13
- package/src/workbench.ts +13 -26
package/src/_metadata.ts
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build metadata utilities
|
|
3
|
+
* Provides cached access to agentuity.metadata.json
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
8
|
+
import { internal } from './logger/internal';
|
|
9
|
+
|
|
10
|
+
export interface BuildMetadataAgent {
|
|
11
|
+
filename: string;
|
|
12
|
+
id: string;
|
|
13
|
+
agentId: string;
|
|
14
|
+
version: string;
|
|
15
|
+
name: string;
|
|
16
|
+
description?: string;
|
|
17
|
+
projectId?: string;
|
|
18
|
+
schema?: {
|
|
19
|
+
input?: string;
|
|
20
|
+
output?: string;
|
|
21
|
+
};
|
|
22
|
+
evals?: Array<{
|
|
23
|
+
filename: string;
|
|
24
|
+
id: string;
|
|
25
|
+
evalId: string;
|
|
26
|
+
name: string;
|
|
27
|
+
version: string;
|
|
28
|
+
description?: string;
|
|
29
|
+
agentIdentifier?: string;
|
|
30
|
+
projectId?: string;
|
|
31
|
+
}>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface BuildMetadataRoute {
|
|
35
|
+
id: string;
|
|
36
|
+
filename: string;
|
|
37
|
+
path: string;
|
|
38
|
+
method: 'get' | 'post' | 'put' | 'delete' | 'patch';
|
|
39
|
+
version: string;
|
|
40
|
+
type?: string;
|
|
41
|
+
agentIds?: string[];
|
|
42
|
+
config?: Record<string, unknown>;
|
|
43
|
+
schema?: {
|
|
44
|
+
input?: string;
|
|
45
|
+
output?: string;
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface BuildMetadata {
|
|
50
|
+
routes: BuildMetadataRoute[];
|
|
51
|
+
agents: BuildMetadataAgent[];
|
|
52
|
+
assets?: string[];
|
|
53
|
+
project: {
|
|
54
|
+
id: string;
|
|
55
|
+
name: string;
|
|
56
|
+
version?: string;
|
|
57
|
+
description?: string;
|
|
58
|
+
keywords?: string[];
|
|
59
|
+
orgId?: string;
|
|
60
|
+
};
|
|
61
|
+
deployment: {
|
|
62
|
+
id: string;
|
|
63
|
+
date: string;
|
|
64
|
+
build: {
|
|
65
|
+
bun: string;
|
|
66
|
+
agentuity: string;
|
|
67
|
+
arch: string;
|
|
68
|
+
platform: string;
|
|
69
|
+
};
|
|
70
|
+
git?: {
|
|
71
|
+
branch?: string;
|
|
72
|
+
repo?: string;
|
|
73
|
+
provider?: string;
|
|
74
|
+
tags?: string[];
|
|
75
|
+
commit?: string;
|
|
76
|
+
message?: string;
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Cached metadata - null means not yet loaded, undefined means file not found
|
|
82
|
+
let _metadataCache: BuildMetadata | null | undefined = null;
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Get the path to agentuity.metadata.json
|
|
86
|
+
*/
|
|
87
|
+
export function getMetadataPath(): string {
|
|
88
|
+
return join(process.cwd(), '.agentuity', 'agentuity.metadata.json');
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Load and cache the build metadata from agentuity.metadata.json
|
|
93
|
+
* Returns undefined if the file doesn't exist or can't be parsed
|
|
94
|
+
*/
|
|
95
|
+
export function loadBuildMetadata(): BuildMetadata | undefined {
|
|
96
|
+
// Return cached value if already loaded
|
|
97
|
+
if (_metadataCache !== null) {
|
|
98
|
+
internal.info(
|
|
99
|
+
'[metadata] loadBuildMetadata: returning cached value (exists: %s)',
|
|
100
|
+
_metadataCache !== undefined
|
|
101
|
+
);
|
|
102
|
+
return _metadataCache;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const metadataPath = getMetadataPath();
|
|
106
|
+
internal.info('[metadata] loadBuildMetadata: checking path %s', metadataPath);
|
|
107
|
+
internal.info('[metadata] loadBuildMetadata: cwd=%s', process.cwd());
|
|
108
|
+
|
|
109
|
+
if (!existsSync(metadataPath)) {
|
|
110
|
+
internal.info('[metadata] agentuity.metadata.json not found at %s', metadataPath);
|
|
111
|
+
_metadataCache = undefined;
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
try {
|
|
116
|
+
internal.info('[metadata] loadBuildMetadata: file exists, reading...');
|
|
117
|
+
const content = readFileSync(metadataPath, 'utf-8');
|
|
118
|
+
const metadata = JSON.parse(content) as BuildMetadata;
|
|
119
|
+
_metadataCache = metadata;
|
|
120
|
+
|
|
121
|
+
// Log agent and eval counts
|
|
122
|
+
let totalEvals = 0;
|
|
123
|
+
for (const agent of metadata.agents ?? []) {
|
|
124
|
+
totalEvals += agent.evals?.length ?? 0;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
internal.info(
|
|
128
|
+
'[metadata] loaded agentuity.metadata.json: %d agents, %d routes, %d total evals',
|
|
129
|
+
metadata.agents?.length ?? 0,
|
|
130
|
+
metadata.routes?.length ?? 0,
|
|
131
|
+
totalEvals
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
// Log agent names and their eval counts
|
|
135
|
+
for (const agent of metadata.agents ?? []) {
|
|
136
|
+
internal.info('[metadata] agent: %s (evals: %d)', agent.name, agent.evals?.length ?? 0);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return metadata;
|
|
140
|
+
} catch (err) {
|
|
141
|
+
internal.info('[metadata] failed to load agentuity.metadata.json: %s', err);
|
|
142
|
+
_metadataCache = undefined;
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Eval metadata type (extracted from agent's evals array)
|
|
148
|
+
export type BuildMetadataEval = NonNullable<BuildMetadataAgent['evals']>[number];
|
|
149
|
+
|
|
150
|
+
// Agent lookup cache - built lazily from metadata
|
|
151
|
+
let _agentsByName: Map<string, BuildMetadataAgent> | null = null;
|
|
152
|
+
let _agentsByAgentId: Map<string, BuildMetadataAgent> | null = null;
|
|
153
|
+
|
|
154
|
+
// Eval lookup cache - nested map: agentName -> evalName -> evalMetadata
|
|
155
|
+
let _evalsByAgentName: Map<string, Map<string, BuildMetadataEval>> | null = null;
|
|
156
|
+
let _evalsByAgentId: Map<string, Map<string, BuildMetadataEval>> | null = null;
|
|
157
|
+
|
|
158
|
+
// Track if we've already attempted a reload for empty eval map
|
|
159
|
+
let _evalReloadAttempted = false;
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Build agent lookup maps from metadata
|
|
163
|
+
*/
|
|
164
|
+
function ensureAgentMaps(): void {
|
|
165
|
+
if (_agentsByName !== null) {
|
|
166
|
+
internal.info(`[metadata] ensureAgentMaps: already initialized, skipping`);
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
internal.info(`[metadata] ensureAgentMaps: initializing agent and eval maps`);
|
|
171
|
+
|
|
172
|
+
_agentsByName = new Map();
|
|
173
|
+
_agentsByAgentId = new Map();
|
|
174
|
+
_evalsByAgentName = new Map();
|
|
175
|
+
_evalsByAgentId = new Map();
|
|
176
|
+
|
|
177
|
+
const metadata = loadBuildMetadata();
|
|
178
|
+
if (!metadata?.agents) {
|
|
179
|
+
internal.info(`[metadata] ensureAgentMaps: no metadata or no agents found`);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
internal.info(`[metadata] ensureAgentMaps: processing ${metadata.agents.length} agents`);
|
|
184
|
+
|
|
185
|
+
for (const agent of metadata.agents) {
|
|
186
|
+
if (agent.name) {
|
|
187
|
+
_agentsByName.set(agent.name, agent);
|
|
188
|
+
}
|
|
189
|
+
if (agent.agentId) {
|
|
190
|
+
_agentsByAgentId.set(agent.agentId, agent);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Build eval lookup maps
|
|
194
|
+
if (agent.evals && agent.evals.length > 0) {
|
|
195
|
+
const evalsByName = new Map<string, BuildMetadataEval>();
|
|
196
|
+
for (const evalMeta of agent.evals) {
|
|
197
|
+
if (evalMeta.name) {
|
|
198
|
+
evalsByName.set(evalMeta.name, evalMeta);
|
|
199
|
+
internal.info(
|
|
200
|
+
`[metadata] Indexed eval: agent='${agent.name}' eval='${evalMeta.name}' evalId='${evalMeta.evalId}'`
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (agent.name) {
|
|
205
|
+
_evalsByAgentName.set(agent.name, evalsByName);
|
|
206
|
+
}
|
|
207
|
+
if (agent.agentId) {
|
|
208
|
+
_evalsByAgentId.set(agent.agentId, evalsByName);
|
|
209
|
+
}
|
|
210
|
+
} else {
|
|
211
|
+
internal.info(`[metadata] Agent '${agent.name}' has no evals`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
internal.info(`[metadata] Eval maps built: ${_evalsByAgentName?.size ?? 0} agents with evals`);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Look up agent metadata by name
|
|
219
|
+
*/
|
|
220
|
+
export function getAgentMetadataByName(agentName: string): BuildMetadataAgent | undefined {
|
|
221
|
+
ensureAgentMaps();
|
|
222
|
+
return _agentsByName?.get(agentName);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Look up agent metadata by agentId
|
|
227
|
+
*/
|
|
228
|
+
export function getAgentMetadataByAgentId(agentId: string): BuildMetadataAgent | undefined {
|
|
229
|
+
ensureAgentMaps();
|
|
230
|
+
return _agentsByAgentId?.get(agentId);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Look up eval metadata by agent name and eval name
|
|
235
|
+
*/
|
|
236
|
+
export function getEvalMetadata(
|
|
237
|
+
agentName: string,
|
|
238
|
+
evalName: string
|
|
239
|
+
): BuildMetadataEval | undefined {
|
|
240
|
+
ensureAgentMaps();
|
|
241
|
+
|
|
242
|
+
// If eval map is empty, the cache may have been built before metadata was ready
|
|
243
|
+
// Try clearing and reloading once (only attempt once to avoid repeated reloads)
|
|
244
|
+
if (_evalsByAgentName?.size === 0 && !_evalReloadAttempted) {
|
|
245
|
+
_evalReloadAttempted = true;
|
|
246
|
+
internal.info(
|
|
247
|
+
`[metadata] getEvalMetadata: eval map is empty, attempting cache clear and reload`
|
|
248
|
+
);
|
|
249
|
+
clearMetadataCache();
|
|
250
|
+
ensureAgentMaps();
|
|
251
|
+
internal.info(
|
|
252
|
+
`[metadata] getEvalMetadata: after reload, eval map size: ${_evalsByAgentName?.size ?? 0}`
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const agentEvals = _evalsByAgentName?.get(agentName);
|
|
257
|
+
internal.info(
|
|
258
|
+
`[metadata] getEvalMetadata('${agentName}', '${evalName}'): agentEvals=${agentEvals ? `Map(${agentEvals.size})` : 'undefined'}`
|
|
259
|
+
);
|
|
260
|
+
if (agentEvals) {
|
|
261
|
+
internal.info(
|
|
262
|
+
`[metadata] Available evals for agent '${agentName}': [${[...agentEvals.keys()].join(', ')}]`
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
if (!agentEvals) {
|
|
266
|
+
internal.info(
|
|
267
|
+
`[metadata] Available agents in eval map: [${[...(_evalsByAgentName?.keys() ?? [])].join(', ')}]`
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
const result = agentEvals?.get(evalName);
|
|
271
|
+
internal.info(
|
|
272
|
+
`[metadata] getEvalMetadata result: ${result ? `found evalId=${result.evalId}` : 'not found'}`
|
|
273
|
+
);
|
|
274
|
+
return result;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Look up eval metadata by agentId and eval name
|
|
279
|
+
*/
|
|
280
|
+
export function getEvalMetadataByAgentId(
|
|
281
|
+
agentId: string,
|
|
282
|
+
evalName: string
|
|
283
|
+
): BuildMetadataEval | undefined {
|
|
284
|
+
ensureAgentMaps();
|
|
285
|
+
return _evalsByAgentId?.get(agentId)?.get(evalName);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Check if metadata file exists (uses cache)
|
|
290
|
+
*/
|
|
291
|
+
export function hasMetadata(): boolean {
|
|
292
|
+
return loadBuildMetadata() !== undefined;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Clear the metadata cache (useful for testing or hot reload)
|
|
297
|
+
*/
|
|
298
|
+
export function clearMetadataCache(): void {
|
|
299
|
+
internal.info('[metadata] clearMetadataCache: clearing all caches');
|
|
300
|
+
_metadataCache = null;
|
|
301
|
+
_agentsByName = null;
|
|
302
|
+
_agentsByAgentId = null;
|
|
303
|
+
_evalsByAgentName = null;
|
|
304
|
+
_evalsByAgentId = null;
|
|
305
|
+
// Note: _evalReloadAttempted is intentionally NOT reset here
|
|
306
|
+
// to prevent infinite reload loops in getEvalMetadata
|
|
307
|
+
}
|
|
@@ -31,6 +31,9 @@ export function enableProcessExitProtection(): void {
|
|
|
31
31
|
|
|
32
32
|
protectionEnabled = true;
|
|
33
33
|
|
|
34
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
35
|
+
(globalThis as any).AGENTUITY_PROCESS_EXIT = originalExit;
|
|
36
|
+
|
|
34
37
|
// Replace process.exit with a function that throws
|
|
35
38
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
36
39
|
(process as any).exit = function (code?: number | string | null | undefined): never {
|
|
@@ -46,6 +49,9 @@ export function disableProcessExitProtection(): void {
|
|
|
46
49
|
return;
|
|
47
50
|
}
|
|
48
51
|
|
|
52
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
53
|
+
(globalThis as any).AGENTUITY_PROCESS_EXIT = undefined;
|
|
54
|
+
|
|
49
55
|
protectionEnabled = false;
|
|
50
56
|
process.exit = originalExit;
|
|
51
57
|
}
|
package/src/_services.ts
CHANGED
|
@@ -50,15 +50,16 @@ import {
|
|
|
50
50
|
} from './services/local';
|
|
51
51
|
|
|
52
52
|
const userAgent = `Agentuity SDK/${getSDKVersion()}`;
|
|
53
|
-
const sdkKey = process.env.AGENTUITY_SDK_KEY;
|
|
54
|
-
const bearerKey = `Bearer ${sdkKey}`;
|
|
55
53
|
|
|
56
|
-
|
|
57
|
-
const
|
|
58
|
-
const
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
const
|
|
54
|
+
// Lazy getters - these must be functions to read env vars AFTER bootstrapRuntimeEnv() runs
|
|
55
|
+
const getSdkKey = () => process.env.AGENTUITY_SDK_KEY;
|
|
56
|
+
const getBearerKey = () => `Bearer ${getSdkKey()}`;
|
|
57
|
+
const getRegion = () => process.env.AGENTUITY_REGION ?? 'usc';
|
|
58
|
+
const getLazyServiceUrls = () => getServiceUrls(getRegion());
|
|
59
|
+
const getKvBaseUrl = () => getLazyServiceUrls().keyvalue;
|
|
60
|
+
const getStreamBaseUrl = () => getLazyServiceUrls().stream;
|
|
61
|
+
const getVectorBaseUrl = () => getLazyServiceUrls().vector;
|
|
62
|
+
const getCatalystBaseUrl = () => getLazyServiceUrls().catalyst;
|
|
62
63
|
|
|
63
64
|
let adapter: FetchAdapter;
|
|
64
65
|
|
|
@@ -66,7 +67,7 @@ const createFetchAdapter = (logger: Logger) =>
|
|
|
66
67
|
createServerFetchAdapter(
|
|
67
68
|
{
|
|
68
69
|
headers: {
|
|
69
|
-
Authorization:
|
|
70
|
+
Authorization: getBearerKey(),
|
|
70
71
|
'User-Agent': userAgent,
|
|
71
72
|
},
|
|
72
73
|
onBefore: async (url, options, callback) => {
|
|
@@ -115,7 +116,7 @@ const createFetchAdapter = (logger: Logger) =>
|
|
|
115
116
|
const res = result.data as { id: string };
|
|
116
117
|
span?.setAttributes({
|
|
117
118
|
'stream.id': res.id,
|
|
118
|
-
'stream.url': `${
|
|
119
|
+
'stream.url': `${getStreamBaseUrl()}/${res.id}`,
|
|
119
120
|
});
|
|
120
121
|
}
|
|
121
122
|
break;
|
|
@@ -225,30 +226,31 @@ export function createServices(logger: Logger, config?: AppConfig<any>, serverUr
|
|
|
225
226
|
localRouter = null;
|
|
226
227
|
|
|
227
228
|
// At this point we must be authenticated (since !authenticated would trigger local services above)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
229
|
+
const catalystUrl = getCatalystBaseUrl();
|
|
230
|
+
kv = config?.services?.keyvalue || new KeyValueStorageService(getKvBaseUrl(), adapter);
|
|
231
|
+
stream = config?.services?.stream || new StreamStorageService(getStreamBaseUrl(), adapter);
|
|
232
|
+
vector = config?.services?.vector || new VectorStorageService(getVectorBaseUrl(), adapter);
|
|
231
233
|
session = config?.services?.session || new DefaultSessionProvider();
|
|
232
234
|
thread = config?.services?.thread || new DefaultThreadProvider();
|
|
233
235
|
// FIXME: this is turned off for now for production until we have the new changes deployed
|
|
234
236
|
sessionEvent =
|
|
235
237
|
isProduction() && process.env.AGENTUITY_CLOUD_EXPORT_DIR
|
|
236
238
|
? new JSONSessionEventProvider(process.env.AGENTUITY_CLOUD_EXPORT_DIR)
|
|
237
|
-
: new HTTPSessionEventProvider(new APIClient(
|
|
238
|
-
new LocalSessionEventProvider();
|
|
239
|
+
: new HTTPSessionEventProvider(new APIClient(catalystUrl, logger), logger);
|
|
239
240
|
if (config?.services?.sessionEvent) {
|
|
240
241
|
sessionEvent = new CompositeSessionEventProvider(sessionEvent, config.services.sessionEvent);
|
|
241
242
|
}
|
|
242
243
|
// FIXME: this is turned off for now for production until we have the new changes deployed
|
|
244
|
+
logger.debug(
|
|
245
|
+
'[SERVICES] Initializing eval run provider - region: %s, catalystBaseUrl: %s, isProduction: %s',
|
|
246
|
+
getRegion(),
|
|
247
|
+
catalystUrl,
|
|
248
|
+
isProduction()
|
|
249
|
+
);
|
|
243
250
|
evalRunEvent =
|
|
244
251
|
isProduction() && process.env.AGENTUITY_CLOUD_EXPORT_DIR
|
|
245
252
|
? new JSONEvalRunEventProvider(process.env.AGENTUITY_CLOUD_EXPORT_DIR)
|
|
246
|
-
: new HTTPEvalRunEventProvider(
|
|
247
|
-
new APIClient(catalystBaseUrl, logger),
|
|
248
|
-
logger,
|
|
249
|
-
catalystBaseUrl
|
|
250
|
-
);
|
|
251
|
-
new LocalEvalRunEventProvider();
|
|
253
|
+
: new HTTPEvalRunEventProvider(new APIClient(catalystUrl, logger), logger, catalystUrl);
|
|
252
254
|
if (config?.services?.evalRunEvent) {
|
|
253
255
|
evalRunEvent = new CompositeEvalRunEventProvider(evalRunEvent, config.services.evalRunEvent);
|
|
254
256
|
}
|
package/src/_standalone.ts
CHANGED
|
@@ -143,6 +143,7 @@ export class StandaloneAgentContext<
|
|
|
143
143
|
({
|
|
144
144
|
id: 'pending',
|
|
145
145
|
state: new Map(),
|
|
146
|
+
metadata: {},
|
|
146
147
|
addEventListener: () => {},
|
|
147
148
|
removeEventListener: () => {},
|
|
148
149
|
destroy: async () => {},
|
|
@@ -155,6 +156,7 @@ export class StandaloneAgentContext<
|
|
|
155
156
|
id: 'pending',
|
|
156
157
|
thread: this.thread,
|
|
157
158
|
state: new Map(),
|
|
159
|
+
metadata: {},
|
|
158
160
|
addEventListener: () => {},
|
|
159
161
|
removeEventListener: () => {},
|
|
160
162
|
serializeUserData: () => undefined,
|
|
@@ -317,6 +319,10 @@ export class StandaloneAgentContext<
|
|
|
317
319
|
method: 'STANDALONE',
|
|
318
320
|
url: '',
|
|
319
321
|
trigger: this.trigger,
|
|
322
|
+
metadata:
|
|
323
|
+
Object.keys(invocationSession.metadata).length > 0
|
|
324
|
+
? invocationSession.metadata
|
|
325
|
+
: undefined,
|
|
320
326
|
})
|
|
321
327
|
.catch((ex) => {
|
|
322
328
|
canSendSessionEvents = false;
|
|
@@ -352,6 +358,10 @@ export class StandaloneAgentContext<
|
|
|
352
358
|
statusCode: 200, // Success
|
|
353
359
|
agentIds: Array.from(agentIds),
|
|
354
360
|
userData,
|
|
361
|
+
metadata:
|
|
362
|
+
Object.keys(invocationSession.metadata).length > 0
|
|
363
|
+
? invocationSession.metadata
|
|
364
|
+
: undefined,
|
|
355
365
|
})
|
|
356
366
|
.then(() => {})
|
|
357
367
|
.catch((ex) => this.logger.error(ex));
|
|
@@ -382,6 +392,10 @@ export class StandaloneAgentContext<
|
|
|
382
392
|
error: message,
|
|
383
393
|
agentIds: Array.from(agentIds),
|
|
384
394
|
userData,
|
|
395
|
+
metadata:
|
|
396
|
+
Object.keys(invocationSession.metadata).length > 0
|
|
397
|
+
? invocationSession.metadata
|
|
398
|
+
: undefined,
|
|
385
399
|
})
|
|
386
400
|
.then(() => {})
|
|
387
401
|
.catch((ex) => this.logger.error(ex));
|
|
@@ -401,6 +415,10 @@ export class StandaloneAgentContext<
|
|
|
401
415
|
statusCode: 200,
|
|
402
416
|
agentIds: Array.from(agentIds),
|
|
403
417
|
userData,
|
|
418
|
+
metadata:
|
|
419
|
+
Object.keys(invocationSession.metadata).length > 0
|
|
420
|
+
? invocationSession.metadata
|
|
421
|
+
: undefined,
|
|
404
422
|
})
|
|
405
423
|
.then(() => {})
|
|
406
424
|
.catch((ex) => this.logger.error(ex));
|
|
@@ -428,6 +446,10 @@ export class StandaloneAgentContext<
|
|
|
428
446
|
error: message,
|
|
429
447
|
agentIds: Array.from(agentIds),
|
|
430
448
|
userData,
|
|
449
|
+
metadata:
|
|
450
|
+
Object.keys(invocationSession.metadata).length > 0
|
|
451
|
+
? invocationSession.metadata
|
|
452
|
+
: undefined,
|
|
431
453
|
})
|
|
432
454
|
.then(() => {})
|
|
433
455
|
.catch((ex) => this.logger.error(ex));
|
package/src/agent.ts
CHANGED
|
@@ -34,6 +34,7 @@ import { getEvalRunEventProvider } from './_services';
|
|
|
34
34
|
import * as runtimeConfig from './_config';
|
|
35
35
|
import type { AppState } from './index';
|
|
36
36
|
import { validateSchema, formatValidationIssues } from './_validation';
|
|
37
|
+
import { getAgentMetadataByName, getEvalMetadata } from './_metadata';
|
|
37
38
|
|
|
38
39
|
export type AgentEventName = 'started' | 'completed' | 'errored';
|
|
39
40
|
|
|
@@ -1527,6 +1528,7 @@ export function createAgent<
|
|
|
1527
1528
|
'@agentuity/agentInstanceId': agent.metadata.agentId,
|
|
1528
1529
|
'@agentuity/agentDescription': agent.metadata.description,
|
|
1529
1530
|
'@agentuity/agentName': agent.metadata.name,
|
|
1531
|
+
'@agentuity/threadId': agentCtx.thread.id,
|
|
1530
1532
|
};
|
|
1531
1533
|
|
|
1532
1534
|
// Set agent attributes on the current active span
|
|
@@ -1538,15 +1540,15 @@ export function createAgent<
|
|
|
1538
1540
|
if (inHTTPContext()) {
|
|
1539
1541
|
const honoCtx = privateContext(getHTTPContext());
|
|
1540
1542
|
if (honoCtx.var.agentIds) {
|
|
1541
|
-
honoCtx.var.agentIds.add(agent.metadata.id);
|
|
1542
|
-
honoCtx.var.agentIds.add(agent.metadata.agentId);
|
|
1543
|
+
if (agent.metadata.id) honoCtx.var.agentIds.add(agent.metadata.id);
|
|
1544
|
+
if (agent.metadata.agentId) honoCtx.var.agentIds.add(agent.metadata.agentId);
|
|
1543
1545
|
}
|
|
1544
1546
|
} else {
|
|
1545
1547
|
// For standalone contexts, check for AGENT_IDS symbol
|
|
1546
1548
|
const agentIds = (agentCtx as any)[AGENT_IDS] as Set<string> | undefined;
|
|
1547
1549
|
if (agentIds) {
|
|
1548
|
-
agentIds.add(agent.metadata.id);
|
|
1549
|
-
agentIds.add(agent.metadata.agentId);
|
|
1550
|
+
if (agent.metadata.id) agentIds.add(agent.metadata.id);
|
|
1551
|
+
if (agent.metadata.agentId) agentIds.add(agent.metadata.agentId);
|
|
1550
1552
|
}
|
|
1551
1553
|
}
|
|
1552
1554
|
|
|
@@ -1667,7 +1669,7 @@ export function createAgent<
|
|
|
1667
1669
|
|
|
1668
1670
|
// Build metadata - merge user-provided metadata with defaults
|
|
1669
1671
|
// The build plugin injects metadata via config.metadata during AST transformation
|
|
1670
|
-
|
|
1672
|
+
let metadata: Partial<AgentMetadata> = {
|
|
1671
1673
|
// Defaults (used when running without build, e.g., dev mode)
|
|
1672
1674
|
name,
|
|
1673
1675
|
description: config.description,
|
|
@@ -1681,6 +1683,26 @@ export function createAgent<
|
|
|
1681
1683
|
...config.metadata,
|
|
1682
1684
|
};
|
|
1683
1685
|
|
|
1686
|
+
// If id/agentId are empty, try to load from agentuity.metadata.json
|
|
1687
|
+
if (!metadata.id || !metadata.agentId) {
|
|
1688
|
+
const fileMetadata = getAgentMetadataByName(name);
|
|
1689
|
+
if (fileMetadata) {
|
|
1690
|
+
internal.info(
|
|
1691
|
+
'[agent] loaded metadata for "%s" from file: id=%s, agentId=%s',
|
|
1692
|
+
name,
|
|
1693
|
+
fileMetadata.id,
|
|
1694
|
+
fileMetadata.agentId
|
|
1695
|
+
);
|
|
1696
|
+
metadata = {
|
|
1697
|
+
...metadata,
|
|
1698
|
+
id: fileMetadata.id || metadata.id,
|
|
1699
|
+
agentId: fileMetadata.agentId || metadata.agentId,
|
|
1700
|
+
filename: fileMetadata.filename || metadata.filename,
|
|
1701
|
+
version: fileMetadata.version || metadata.version,
|
|
1702
|
+
};
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1684
1706
|
const agent: any = {
|
|
1685
1707
|
handler,
|
|
1686
1708
|
metadata,
|
|
@@ -1737,19 +1759,39 @@ export function createAgent<
|
|
|
1737
1759
|
// Execute each eval using waitUntil to avoid blocking the response
|
|
1738
1760
|
for (const evalItem of agentEvals) {
|
|
1739
1761
|
const evalName = evalItem.metadata.name || 'unnamed';
|
|
1762
|
+
const agentName = _agent?.metadata?.name || name;
|
|
1740
1763
|
|
|
1741
1764
|
ctx.waitUntil(
|
|
1742
1765
|
(async () => {
|
|
1743
1766
|
internal.info(`[EVALRUN] Starting eval run tracking for '${evalName}'`);
|
|
1744
1767
|
const evalRunId = generateId('evalrun');
|
|
1745
|
-
|
|
1746
|
-
|
|
1768
|
+
|
|
1769
|
+
// Look up eval metadata from agentuity.metadata.json by agent name and eval name
|
|
1770
|
+
internal.info(
|
|
1771
|
+
`[EVALRUN] Looking up eval metadata: agentName='${agentName}', evalName='${evalName}'`
|
|
1772
|
+
);
|
|
1773
|
+
const evalMeta = getEvalMetadata(agentName, evalName);
|
|
1774
|
+
internal.info(`[EVALRUN] Eval metadata lookup result:`, {
|
|
1775
|
+
found: !!evalMeta,
|
|
1776
|
+
evalId: evalMeta?.evalId,
|
|
1777
|
+
id: evalMeta?.id,
|
|
1778
|
+
filename: evalMeta?.filename,
|
|
1779
|
+
});
|
|
1780
|
+
|
|
1781
|
+
// evalId = deployment-specific ID (evalid_...), evalIdentifier = stable (eval_...)
|
|
1782
|
+
const evalId = evalMeta?.id || '';
|
|
1783
|
+
const evalIdentifier = evalMeta?.evalId || '';
|
|
1784
|
+
internal.info(
|
|
1785
|
+
`[EVALRUN] Resolved evalId='${evalId}', evalIdentifier='${evalIdentifier}'`
|
|
1786
|
+
);
|
|
1747
1787
|
|
|
1748
1788
|
// Log eval metadata using structured logging and tracing
|
|
1749
1789
|
ctx.logger.debug('Starting eval run with metadata', {
|
|
1750
1790
|
evalName,
|
|
1791
|
+
agentName,
|
|
1751
1792
|
evalRunId,
|
|
1752
1793
|
evalId,
|
|
1794
|
+
evalMetaFromFile: !!evalMeta,
|
|
1753
1795
|
evalMetadata: evalItem.metadata,
|
|
1754
1796
|
});
|
|
1755
1797
|
|
|
@@ -1760,8 +1802,9 @@ export function createAgent<
|
|
|
1760
1802
|
'eval.name': evalName,
|
|
1761
1803
|
'eval.id': evalId,
|
|
1762
1804
|
'eval.runId': evalRunId,
|
|
1763
|
-
'eval.description':
|
|
1764
|
-
|
|
1805
|
+
'eval.description':
|
|
1806
|
+
evalMeta?.description || evalItem.metadata.description || '',
|
|
1807
|
+
'eval.filename': evalMeta?.filename || evalItem.metadata.filename || '',
|
|
1765
1808
|
});
|
|
1766
1809
|
}
|
|
1767
1810
|
|
|
@@ -1771,12 +1814,14 @@ export function createAgent<
|
|
|
1771
1814
|
const evalRunEventProvider = getEvalRunEventProvider();
|
|
1772
1815
|
|
|
1773
1816
|
// Only send events if we have required context (devmode flag will be set based on devMode)
|
|
1774
|
-
const shouldSendEvalRunEvents =
|
|
1817
|
+
const shouldSendEvalRunEvents =
|
|
1818
|
+
orgId && projectId && evalId !== '' && evalIdentifier !== '';
|
|
1775
1819
|
|
|
1776
1820
|
internal.info(`[EVALRUN] Checking conditions for eval '${evalName}':`, {
|
|
1777
1821
|
orgId: orgId,
|
|
1778
1822
|
projectId: projectId,
|
|
1779
1823
|
evalId: evalId,
|
|
1824
|
+
evalIdentifier: evalIdentifier,
|
|
1780
1825
|
devMode,
|
|
1781
1826
|
hasEvalRunEventProvider: !!evalRunEventProvider,
|
|
1782
1827
|
shouldSendEvalRunEvents,
|
|
@@ -1787,6 +1832,8 @@ export function createAgent<
|
|
|
1787
1832
|
if (!orgId) reasons.push('missing orgId');
|
|
1788
1833
|
if (!projectId) reasons.push('missing projectId');
|
|
1789
1834
|
if (!evalId || evalId === '') reasons.push('empty evalId');
|
|
1835
|
+
if (!evalIdentifier || evalIdentifier === '')
|
|
1836
|
+
reasons.push('empty evalIdentifier');
|
|
1790
1837
|
internal.info(
|
|
1791
1838
|
`[EVALRUN] Skipping eval run events for '${evalName}': ${reasons.join(', ')}`
|
|
1792
1839
|
);
|
|
@@ -1811,7 +1858,8 @@ export function createAgent<
|
|
|
1811
1858
|
const startEvent: EvalRunStartEvent = {
|
|
1812
1859
|
id: evalRunId,
|
|
1813
1860
|
sessionId: ctx.sessionId,
|
|
1814
|
-
evalId: evalId,
|
|
1861
|
+
evalId: evalId, // deployment-specific ID (evalid_...)
|
|
1862
|
+
evalIdentifier: evalIdentifier, // stable identifier (eval_...)
|
|
1815
1863
|
orgId: orgId!,
|
|
1816
1864
|
projectId: projectId!,
|
|
1817
1865
|
devmode: Boolean(devMode),
|
|
@@ -2140,6 +2188,7 @@ const runWithSpan = async <
|
|
|
2140
2188
|
'@agentuity/agentInstanceId': agent.metadata.agentId,
|
|
2141
2189
|
'@agentuity/agentDescription': agent.metadata.description,
|
|
2142
2190
|
'@agentuity/agentName': agent.metadata.name,
|
|
2191
|
+
'@agentuity/threadId': ctx.var.thread.id,
|
|
2143
2192
|
});
|
|
2144
2193
|
|
|
2145
2194
|
const spanId = span.spanContext().spanId;
|
|
@@ -2244,9 +2293,11 @@ export const createAgentMiddleware = (agentName: AgentName | ''): MiddlewareHand
|
|
|
2244
2293
|
const agentKey = toCamelCase(agentName);
|
|
2245
2294
|
const agent = agentsObj[agentKey];
|
|
2246
2295
|
const _ctx = privateContext(ctx);
|
|
2296
|
+
// we add both so that you can query by either
|
|
2247
2297
|
if (agent?.metadata?.id) {
|
|
2248
|
-
// we add both so that you can query by either
|
|
2249
2298
|
_ctx.var.agentIds.add(agent.metadata.id);
|
|
2299
|
+
}
|
|
2300
|
+
if (agent?.metadata?.agentId) {
|
|
2250
2301
|
_ctx.var.agentIds.add(agent.metadata.agentId);
|
|
2251
2302
|
}
|
|
2252
2303
|
}
|