@agentuity/runtime 0.0.100 → 0.0.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/AGENTS.md +34 -212
  2. package/dist/_metadata.d.ts +107 -0
  3. package/dist/_metadata.d.ts.map +1 -0
  4. package/dist/_metadata.js +179 -0
  5. package/dist/_metadata.js.map +1 -0
  6. package/dist/_process-protection.d.ts.map +1 -1
  7. package/dist/_process-protection.js +4 -0
  8. package/dist/_process-protection.js.map +1 -1
  9. package/dist/_services.d.ts.map +1 -1
  10. package/dist/_services.js +18 -17
  11. package/dist/_services.js.map +1 -1
  12. package/dist/_standalone.d.ts.map +1 -1
  13. package/dist/_standalone.js +17 -0
  14. package/dist/_standalone.js.map +1 -1
  15. package/dist/agent.d.ts.map +1 -1
  16. package/dist/agent.js +53 -12
  17. package/dist/agent.js.map +1 -1
  18. package/dist/app.d.ts +61 -10
  19. package/dist/app.d.ts.map +1 -1
  20. package/dist/app.js.map +1 -1
  21. package/dist/devmode.d.ts.map +1 -1
  22. package/dist/devmode.js +13 -5
  23. package/dist/devmode.js.map +1 -1
  24. package/dist/index.d.ts +3 -2
  25. package/dist/index.d.ts.map +1 -1
  26. package/dist/index.js +5 -1
  27. package/dist/index.js.map +1 -1
  28. package/dist/middleware.d.ts +61 -5
  29. package/dist/middleware.d.ts.map +1 -1
  30. package/dist/middleware.js +192 -25
  31. package/dist/middleware.js.map +1 -1
  32. package/dist/services/evalrun/http.d.ts.map +1 -1
  33. package/dist/services/evalrun/http.js +14 -4
  34. package/dist/services/evalrun/http.js.map +1 -1
  35. package/dist/services/local/vector.d.ts +5 -1
  36. package/dist/services/local/vector.d.ts.map +1 -1
  37. package/dist/services/local/vector.js +112 -0
  38. package/dist/services/local/vector.js.map +1 -1
  39. package/dist/services/session/http.d.ts.map +1 -1
  40. package/dist/services/session/http.js +7 -0
  41. package/dist/services/session/http.js.map +1 -1
  42. package/dist/services/session/local.d.ts +2 -2
  43. package/dist/services/session/local.d.ts.map +1 -1
  44. package/dist/services/session/local.js +5 -4
  45. package/dist/services/session/local.js.map +1 -1
  46. package/dist/session.d.ts +30 -4
  47. package/dist/session.d.ts.map +1 -1
  48. package/dist/session.js +90 -13
  49. package/dist/session.js.map +1 -1
  50. package/dist/workbench.d.ts.map +1 -1
  51. package/dist/workbench.js +13 -20
  52. package/dist/workbench.js.map +1 -1
  53. package/package.json +5 -5
  54. package/src/_metadata.ts +307 -0
  55. package/src/_process-protection.ts +6 -0
  56. package/src/_services.ts +23 -21
  57. package/src/_standalone.ts +22 -0
  58. package/src/agent.ts +63 -12
  59. package/src/app.ts +65 -9
  60. package/src/devmode.ts +16 -5
  61. package/src/index.ts +12 -2
  62. package/src/middleware.ts +221 -29
  63. package/src/services/evalrun/http.ts +15 -4
  64. package/src/services/local/vector.ts +160 -0
  65. package/src/services/session/http.ts +11 -0
  66. package/src/services/session/local.ts +9 -4
  67. package/src/session.ts +142 -13
  68. package/src/workbench.ts +13 -26
@@ -0,0 +1,307 @@
1
+ /**
2
+ * Build metadata utilities
3
+ * Provides cached access to agentuity.metadata.json
4
+ */
5
+
6
+ import { join } from 'node:path';
7
+ import { existsSync, readFileSync } from 'node:fs';
8
+ import { internal } from './logger/internal';
9
+
10
+ export interface BuildMetadataAgent {
11
+ filename: string;
12
+ id: string;
13
+ agentId: string;
14
+ version: string;
15
+ name: string;
16
+ description?: string;
17
+ projectId?: string;
18
+ schema?: {
19
+ input?: string;
20
+ output?: string;
21
+ };
22
+ evals?: Array<{
23
+ filename: string;
24
+ id: string;
25
+ evalId: string;
26
+ name: string;
27
+ version: string;
28
+ description?: string;
29
+ agentIdentifier?: string;
30
+ projectId?: string;
31
+ }>;
32
+ }
33
+
34
+ export interface BuildMetadataRoute {
35
+ id: string;
36
+ filename: string;
37
+ path: string;
38
+ method: 'get' | 'post' | 'put' | 'delete' | 'patch';
39
+ version: string;
40
+ type?: string;
41
+ agentIds?: string[];
42
+ config?: Record<string, unknown>;
43
+ schema?: {
44
+ input?: string;
45
+ output?: string;
46
+ };
47
+ }
48
+
49
+ export interface BuildMetadata {
50
+ routes: BuildMetadataRoute[];
51
+ agents: BuildMetadataAgent[];
52
+ assets?: string[];
53
+ project: {
54
+ id: string;
55
+ name: string;
56
+ version?: string;
57
+ description?: string;
58
+ keywords?: string[];
59
+ orgId?: string;
60
+ };
61
+ deployment: {
62
+ id: string;
63
+ date: string;
64
+ build: {
65
+ bun: string;
66
+ agentuity: string;
67
+ arch: string;
68
+ platform: string;
69
+ };
70
+ git?: {
71
+ branch?: string;
72
+ repo?: string;
73
+ provider?: string;
74
+ tags?: string[];
75
+ commit?: string;
76
+ message?: string;
77
+ };
78
+ };
79
+ }
80
+
81
+ // Cached metadata - null means not yet loaded, undefined means file not found
82
+ let _metadataCache: BuildMetadata | null | undefined = null;
83
+
84
+ /**
85
+ * Get the path to agentuity.metadata.json
86
+ */
87
+ export function getMetadataPath(): string {
88
+ return join(process.cwd(), '.agentuity', 'agentuity.metadata.json');
89
+ }
90
+
91
+ /**
92
+ * Load and cache the build metadata from agentuity.metadata.json
93
+ * Returns undefined if the file doesn't exist or can't be parsed
94
+ */
95
+ export function loadBuildMetadata(): BuildMetadata | undefined {
96
+ // Return cached value if already loaded
97
+ if (_metadataCache !== null) {
98
+ internal.info(
99
+ '[metadata] loadBuildMetadata: returning cached value (exists: %s)',
100
+ _metadataCache !== undefined
101
+ );
102
+ return _metadataCache;
103
+ }
104
+
105
+ const metadataPath = getMetadataPath();
106
+ internal.info('[metadata] loadBuildMetadata: checking path %s', metadataPath);
107
+ internal.info('[metadata] loadBuildMetadata: cwd=%s', process.cwd());
108
+
109
+ if (!existsSync(metadataPath)) {
110
+ internal.info('[metadata] agentuity.metadata.json not found at %s', metadataPath);
111
+ _metadataCache = undefined;
112
+ return undefined;
113
+ }
114
+
115
+ try {
116
+ internal.info('[metadata] loadBuildMetadata: file exists, reading...');
117
+ const content = readFileSync(metadataPath, 'utf-8');
118
+ const metadata = JSON.parse(content) as BuildMetadata;
119
+ _metadataCache = metadata;
120
+
121
+ // Log agent and eval counts
122
+ let totalEvals = 0;
123
+ for (const agent of metadata.agents ?? []) {
124
+ totalEvals += agent.evals?.length ?? 0;
125
+ }
126
+
127
+ internal.info(
128
+ '[metadata] loaded agentuity.metadata.json: %d agents, %d routes, %d total evals',
129
+ metadata.agents?.length ?? 0,
130
+ metadata.routes?.length ?? 0,
131
+ totalEvals
132
+ );
133
+
134
+ // Log agent names and their eval counts
135
+ for (const agent of metadata.agents ?? []) {
136
+ internal.info('[metadata] agent: %s (evals: %d)', agent.name, agent.evals?.length ?? 0);
137
+ }
138
+
139
+ return metadata;
140
+ } catch (err) {
141
+ internal.info('[metadata] failed to load agentuity.metadata.json: %s', err);
142
+ _metadataCache = undefined;
143
+ return undefined;
144
+ }
145
+ }
146
+
147
+ // Eval metadata type (extracted from agent's evals array)
148
+ export type BuildMetadataEval = NonNullable<BuildMetadataAgent['evals']>[number];
149
+
150
+ // Agent lookup cache - built lazily from metadata
151
+ let _agentsByName: Map<string, BuildMetadataAgent> | null = null;
152
+ let _agentsByAgentId: Map<string, BuildMetadataAgent> | null = null;
153
+
154
+ // Eval lookup cache - nested map: agentName -> evalName -> evalMetadata
155
+ let _evalsByAgentName: Map<string, Map<string, BuildMetadataEval>> | null = null;
156
+ let _evalsByAgentId: Map<string, Map<string, BuildMetadataEval>> | null = null;
157
+
158
+ // Track if we've already attempted a reload for empty eval map
159
+ let _evalReloadAttempted = false;
160
+
161
+ /**
162
+ * Build agent lookup maps from metadata
163
+ */
164
+ function ensureAgentMaps(): void {
165
+ if (_agentsByName !== null) {
166
+ internal.info(`[metadata] ensureAgentMaps: already initialized, skipping`);
167
+ return;
168
+ }
169
+
170
+ internal.info(`[metadata] ensureAgentMaps: initializing agent and eval maps`);
171
+
172
+ _agentsByName = new Map();
173
+ _agentsByAgentId = new Map();
174
+ _evalsByAgentName = new Map();
175
+ _evalsByAgentId = new Map();
176
+
177
+ const metadata = loadBuildMetadata();
178
+ if (!metadata?.agents) {
179
+ internal.info(`[metadata] ensureAgentMaps: no metadata or no agents found`);
180
+ return;
181
+ }
182
+
183
+ internal.info(`[metadata] ensureAgentMaps: processing ${metadata.agents.length} agents`);
184
+
185
+ for (const agent of metadata.agents) {
186
+ if (agent.name) {
187
+ _agentsByName.set(agent.name, agent);
188
+ }
189
+ if (agent.agentId) {
190
+ _agentsByAgentId.set(agent.agentId, agent);
191
+ }
192
+
193
+ // Build eval lookup maps
194
+ if (agent.evals && agent.evals.length > 0) {
195
+ const evalsByName = new Map<string, BuildMetadataEval>();
196
+ for (const evalMeta of agent.evals) {
197
+ if (evalMeta.name) {
198
+ evalsByName.set(evalMeta.name, evalMeta);
199
+ internal.info(
200
+ `[metadata] Indexed eval: agent='${agent.name}' eval='${evalMeta.name}' evalId='${evalMeta.evalId}'`
201
+ );
202
+ }
203
+ }
204
+ if (agent.name) {
205
+ _evalsByAgentName.set(agent.name, evalsByName);
206
+ }
207
+ if (agent.agentId) {
208
+ _evalsByAgentId.set(agent.agentId, evalsByName);
209
+ }
210
+ } else {
211
+ internal.info(`[metadata] Agent '${agent.name}' has no evals`);
212
+ }
213
+ }
214
+ internal.info(`[metadata] Eval maps built: ${_evalsByAgentName?.size ?? 0} agents with evals`);
215
+ }
216
+
217
+ /**
218
+ * Look up agent metadata by name
219
+ */
220
+ export function getAgentMetadataByName(agentName: string): BuildMetadataAgent | undefined {
221
+ ensureAgentMaps();
222
+ return _agentsByName?.get(agentName);
223
+ }
224
+
225
+ /**
226
+ * Look up agent metadata by agentId
227
+ */
228
+ export function getAgentMetadataByAgentId(agentId: string): BuildMetadataAgent | undefined {
229
+ ensureAgentMaps();
230
+ return _agentsByAgentId?.get(agentId);
231
+ }
232
+
233
+ /**
234
+ * Look up eval metadata by agent name and eval name
235
+ */
236
+ export function getEvalMetadata(
237
+ agentName: string,
238
+ evalName: string
239
+ ): BuildMetadataEval | undefined {
240
+ ensureAgentMaps();
241
+
242
+ // If eval map is empty, the cache may have been built before metadata was ready
243
+ // Try clearing and reloading once (only attempt once to avoid repeated reloads)
244
+ if (_evalsByAgentName?.size === 0 && !_evalReloadAttempted) {
245
+ _evalReloadAttempted = true;
246
+ internal.info(
247
+ `[metadata] getEvalMetadata: eval map is empty, attempting cache clear and reload`
248
+ );
249
+ clearMetadataCache();
250
+ ensureAgentMaps();
251
+ internal.info(
252
+ `[metadata] getEvalMetadata: after reload, eval map size: ${_evalsByAgentName?.size ?? 0}`
253
+ );
254
+ }
255
+
256
+ const agentEvals = _evalsByAgentName?.get(agentName);
257
+ internal.info(
258
+ `[metadata] getEvalMetadata('${agentName}', '${evalName}'): agentEvals=${agentEvals ? `Map(${agentEvals.size})` : 'undefined'}`
259
+ );
260
+ if (agentEvals) {
261
+ internal.info(
262
+ `[metadata] Available evals for agent '${agentName}': [${[...agentEvals.keys()].join(', ')}]`
263
+ );
264
+ }
265
+ if (!agentEvals) {
266
+ internal.info(
267
+ `[metadata] Available agents in eval map: [${[...(_evalsByAgentName?.keys() ?? [])].join(', ')}]`
268
+ );
269
+ }
270
+ const result = agentEvals?.get(evalName);
271
+ internal.info(
272
+ `[metadata] getEvalMetadata result: ${result ? `found evalId=${result.evalId}` : 'not found'}`
273
+ );
274
+ return result;
275
+ }
276
+
277
+ /**
278
+ * Look up eval metadata by agentId and eval name
279
+ */
280
+ export function getEvalMetadataByAgentId(
281
+ agentId: string,
282
+ evalName: string
283
+ ): BuildMetadataEval | undefined {
284
+ ensureAgentMaps();
285
+ return _evalsByAgentId?.get(agentId)?.get(evalName);
286
+ }
287
+
288
+ /**
289
+ * Check if metadata file exists (uses cache)
290
+ */
291
+ export function hasMetadata(): boolean {
292
+ return loadBuildMetadata() !== undefined;
293
+ }
294
+
295
+ /**
296
+ * Clear the metadata cache (useful for testing or hot reload)
297
+ */
298
+ export function clearMetadataCache(): void {
299
+ internal.info('[metadata] clearMetadataCache: clearing all caches');
300
+ _metadataCache = null;
301
+ _agentsByName = null;
302
+ _agentsByAgentId = null;
303
+ _evalsByAgentName = null;
304
+ _evalsByAgentId = null;
305
+ // Note: _evalReloadAttempted is intentionally NOT reset here
306
+ // to prevent infinite reload loops in getEvalMetadata
307
+ }
@@ -31,6 +31,9 @@ export function enableProcessExitProtection(): void {
31
31
 
32
32
  protectionEnabled = true;
33
33
 
34
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
35
+ (globalThis as any).AGENTUITY_PROCESS_EXIT = originalExit;
36
+
34
37
  // Replace process.exit with a function that throws
35
38
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
36
39
  (process as any).exit = function (code?: number | string | null | undefined): never {
@@ -46,6 +49,9 @@ export function disableProcessExitProtection(): void {
46
49
  return;
47
50
  }
48
51
 
52
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
53
+ (globalThis as any).AGENTUITY_PROCESS_EXIT = undefined;
54
+
49
55
  protectionEnabled = false;
50
56
  process.exit = originalExit;
51
57
  }
package/src/_services.ts CHANGED
@@ -50,15 +50,16 @@ import {
50
50
  } from './services/local';
51
51
 
52
52
  const userAgent = `Agentuity SDK/${getSDKVersion()}`;
53
- const sdkKey = process.env.AGENTUITY_SDK_KEY;
54
- const bearerKey = `Bearer ${sdkKey}`;
55
53
 
56
- const region = process.env.AGENTUITY_REGION ?? 'usc';
57
- const serviceUrls = getServiceUrls(region);
58
- const kvBaseUrl = serviceUrls.keyvalue;
59
- const streamBaseUrl = serviceUrls.stream;
60
- const vectorBaseUrl = serviceUrls.vector;
61
- const catalystBaseUrl = serviceUrls.catalyst;
54
+ // Lazy getters - these must be functions to read env vars AFTER bootstrapRuntimeEnv() runs
55
+ const getSdkKey = () => process.env.AGENTUITY_SDK_KEY;
56
+ const getBearerKey = () => `Bearer ${getSdkKey()}`;
57
+ const getRegion = () => process.env.AGENTUITY_REGION ?? 'usc';
58
+ const getLazyServiceUrls = () => getServiceUrls(getRegion());
59
+ const getKvBaseUrl = () => getLazyServiceUrls().keyvalue;
60
+ const getStreamBaseUrl = () => getLazyServiceUrls().stream;
61
+ const getVectorBaseUrl = () => getLazyServiceUrls().vector;
62
+ const getCatalystBaseUrl = () => getLazyServiceUrls().catalyst;
62
63
 
63
64
  let adapter: FetchAdapter;
64
65
 
@@ -66,7 +67,7 @@ const createFetchAdapter = (logger: Logger) =>
66
67
  createServerFetchAdapter(
67
68
  {
68
69
  headers: {
69
- Authorization: bearerKey,
70
+ Authorization: getBearerKey(),
70
71
  'User-Agent': userAgent,
71
72
  },
72
73
  onBefore: async (url, options, callback) => {
@@ -115,7 +116,7 @@ const createFetchAdapter = (logger: Logger) =>
115
116
  const res = result.data as { id: string };
116
117
  span?.setAttributes({
117
118
  'stream.id': res.id,
118
- 'stream.url': `${streamBaseUrl}/${res.id}`,
119
+ 'stream.url': `${getStreamBaseUrl()}/${res.id}`,
119
120
  });
120
121
  }
121
122
  break;
@@ -225,30 +226,31 @@ export function createServices(logger: Logger, config?: AppConfig<any>, serverUr
225
226
  localRouter = null;
226
227
 
227
228
  // At this point we must be authenticated (since !authenticated would trigger local services above)
228
- kv = config?.services?.keyvalue || new KeyValueStorageService(kvBaseUrl, adapter);
229
- stream = config?.services?.stream || new StreamStorageService(streamBaseUrl, adapter);
230
- vector = config?.services?.vector || new VectorStorageService(vectorBaseUrl, adapter);
229
+ const catalystUrl = getCatalystBaseUrl();
230
+ kv = config?.services?.keyvalue || new KeyValueStorageService(getKvBaseUrl(), adapter);
231
+ stream = config?.services?.stream || new StreamStorageService(getStreamBaseUrl(), adapter);
232
+ vector = config?.services?.vector || new VectorStorageService(getVectorBaseUrl(), adapter);
231
233
  session = config?.services?.session || new DefaultSessionProvider();
232
234
  thread = config?.services?.thread || new DefaultThreadProvider();
233
235
  // FIXME: this is turned off for now for production until we have the new changes deployed
234
236
  sessionEvent =
235
237
  isProduction() && process.env.AGENTUITY_CLOUD_EXPORT_DIR
236
238
  ? new JSONSessionEventProvider(process.env.AGENTUITY_CLOUD_EXPORT_DIR)
237
- : new HTTPSessionEventProvider(new APIClient(catalystBaseUrl, logger), logger);
238
- new LocalSessionEventProvider();
239
+ : new HTTPSessionEventProvider(new APIClient(catalystUrl, logger), logger);
239
240
  if (config?.services?.sessionEvent) {
240
241
  sessionEvent = new CompositeSessionEventProvider(sessionEvent, config.services.sessionEvent);
241
242
  }
242
243
  // FIXME: this is turned off for now for production until we have the new changes deployed
244
+ logger.debug(
245
+ '[SERVICES] Initializing eval run provider - region: %s, catalystBaseUrl: %s, isProduction: %s',
246
+ getRegion(),
247
+ catalystUrl,
248
+ isProduction()
249
+ );
243
250
  evalRunEvent =
244
251
  isProduction() && process.env.AGENTUITY_CLOUD_EXPORT_DIR
245
252
  ? new JSONEvalRunEventProvider(process.env.AGENTUITY_CLOUD_EXPORT_DIR)
246
- : new HTTPEvalRunEventProvider(
247
- new APIClient(catalystBaseUrl, logger),
248
- logger,
249
- catalystBaseUrl
250
- );
251
- new LocalEvalRunEventProvider();
253
+ : new HTTPEvalRunEventProvider(new APIClient(catalystUrl, logger), logger, catalystUrl);
252
254
  if (config?.services?.evalRunEvent) {
253
255
  evalRunEvent = new CompositeEvalRunEventProvider(evalRunEvent, config.services.evalRunEvent);
254
256
  }
@@ -143,6 +143,7 @@ export class StandaloneAgentContext<
143
143
  ({
144
144
  id: 'pending',
145
145
  state: new Map(),
146
+ metadata: {},
146
147
  addEventListener: () => {},
147
148
  removeEventListener: () => {},
148
149
  destroy: async () => {},
@@ -155,6 +156,7 @@ export class StandaloneAgentContext<
155
156
  id: 'pending',
156
157
  thread: this.thread,
157
158
  state: new Map(),
159
+ metadata: {},
158
160
  addEventListener: () => {},
159
161
  removeEventListener: () => {},
160
162
  serializeUserData: () => undefined,
@@ -317,6 +319,10 @@ export class StandaloneAgentContext<
317
319
  method: 'STANDALONE',
318
320
  url: '',
319
321
  trigger: this.trigger,
322
+ metadata:
323
+ Object.keys(invocationSession.metadata).length > 0
324
+ ? invocationSession.metadata
325
+ : undefined,
320
326
  })
321
327
  .catch((ex) => {
322
328
  canSendSessionEvents = false;
@@ -352,6 +358,10 @@ export class StandaloneAgentContext<
352
358
  statusCode: 200, // Success
353
359
  agentIds: Array.from(agentIds),
354
360
  userData,
361
+ metadata:
362
+ Object.keys(invocationSession.metadata).length > 0
363
+ ? invocationSession.metadata
364
+ : undefined,
355
365
  })
356
366
  .then(() => {})
357
367
  .catch((ex) => this.logger.error(ex));
@@ -382,6 +392,10 @@ export class StandaloneAgentContext<
382
392
  error: message,
383
393
  agentIds: Array.from(agentIds),
384
394
  userData,
395
+ metadata:
396
+ Object.keys(invocationSession.metadata).length > 0
397
+ ? invocationSession.metadata
398
+ : undefined,
385
399
  })
386
400
  .then(() => {})
387
401
  .catch((ex) => this.logger.error(ex));
@@ -401,6 +415,10 @@ export class StandaloneAgentContext<
401
415
  statusCode: 200,
402
416
  agentIds: Array.from(agentIds),
403
417
  userData,
418
+ metadata:
419
+ Object.keys(invocationSession.metadata).length > 0
420
+ ? invocationSession.metadata
421
+ : undefined,
404
422
  })
405
423
  .then(() => {})
406
424
  .catch((ex) => this.logger.error(ex));
@@ -428,6 +446,10 @@ export class StandaloneAgentContext<
428
446
  error: message,
429
447
  agentIds: Array.from(agentIds),
430
448
  userData,
449
+ metadata:
450
+ Object.keys(invocationSession.metadata).length > 0
451
+ ? invocationSession.metadata
452
+ : undefined,
431
453
  })
432
454
  .then(() => {})
433
455
  .catch((ex) => this.logger.error(ex));
package/src/agent.ts CHANGED
@@ -34,6 +34,7 @@ import { getEvalRunEventProvider } from './_services';
34
34
  import * as runtimeConfig from './_config';
35
35
  import type { AppState } from './index';
36
36
  import { validateSchema, formatValidationIssues } from './_validation';
37
+ import { getAgentMetadataByName, getEvalMetadata } from './_metadata';
37
38
 
38
39
  export type AgentEventName = 'started' | 'completed' | 'errored';
39
40
 
@@ -1527,6 +1528,7 @@ export function createAgent<
1527
1528
  '@agentuity/agentInstanceId': agent.metadata.agentId,
1528
1529
  '@agentuity/agentDescription': agent.metadata.description,
1529
1530
  '@agentuity/agentName': agent.metadata.name,
1531
+ '@agentuity/threadId': agentCtx.thread.id,
1530
1532
  };
1531
1533
 
1532
1534
  // Set agent attributes on the current active span
@@ -1538,15 +1540,15 @@ export function createAgent<
1538
1540
  if (inHTTPContext()) {
1539
1541
  const honoCtx = privateContext(getHTTPContext());
1540
1542
  if (honoCtx.var.agentIds) {
1541
- honoCtx.var.agentIds.add(agent.metadata.id);
1542
- honoCtx.var.agentIds.add(agent.metadata.agentId);
1543
+ if (agent.metadata.id) honoCtx.var.agentIds.add(agent.metadata.id);
1544
+ if (agent.metadata.agentId) honoCtx.var.agentIds.add(agent.metadata.agentId);
1543
1545
  }
1544
1546
  } else {
1545
1547
  // For standalone contexts, check for AGENT_IDS symbol
1546
1548
  const agentIds = (agentCtx as any)[AGENT_IDS] as Set<string> | undefined;
1547
1549
  if (agentIds) {
1548
- agentIds.add(agent.metadata.id);
1549
- agentIds.add(agent.metadata.agentId);
1550
+ if (agent.metadata.id) agentIds.add(agent.metadata.id);
1551
+ if (agent.metadata.agentId) agentIds.add(agent.metadata.agentId);
1550
1552
  }
1551
1553
  }
1552
1554
 
@@ -1667,7 +1669,7 @@ export function createAgent<
1667
1669
 
1668
1670
  // Build metadata - merge user-provided metadata with defaults
1669
1671
  // The build plugin injects metadata via config.metadata during AST transformation
1670
- const metadata: Partial<AgentMetadata> = {
1672
+ let metadata: Partial<AgentMetadata> = {
1671
1673
  // Defaults (used when running without build, e.g., dev mode)
1672
1674
  name,
1673
1675
  description: config.description,
@@ -1681,6 +1683,26 @@ export function createAgent<
1681
1683
  ...config.metadata,
1682
1684
  };
1683
1685
 
1686
+ // If id/agentId are empty, try to load from agentuity.metadata.json
1687
+ if (!metadata.id || !metadata.agentId) {
1688
+ const fileMetadata = getAgentMetadataByName(name);
1689
+ if (fileMetadata) {
1690
+ internal.info(
1691
+ '[agent] loaded metadata for "%s" from file: id=%s, agentId=%s',
1692
+ name,
1693
+ fileMetadata.id,
1694
+ fileMetadata.agentId
1695
+ );
1696
+ metadata = {
1697
+ ...metadata,
1698
+ id: fileMetadata.id || metadata.id,
1699
+ agentId: fileMetadata.agentId || metadata.agentId,
1700
+ filename: fileMetadata.filename || metadata.filename,
1701
+ version: fileMetadata.version || metadata.version,
1702
+ };
1703
+ }
1704
+ }
1705
+
1684
1706
  const agent: any = {
1685
1707
  handler,
1686
1708
  metadata,
@@ -1737,19 +1759,39 @@ export function createAgent<
1737
1759
  // Execute each eval using waitUntil to avoid blocking the response
1738
1760
  for (const evalItem of agentEvals) {
1739
1761
  const evalName = evalItem.metadata.name || 'unnamed';
1762
+ const agentName = _agent?.metadata?.name || name;
1740
1763
 
1741
1764
  ctx.waitUntil(
1742
1765
  (async () => {
1743
1766
  internal.info(`[EVALRUN] Starting eval run tracking for '${evalName}'`);
1744
1767
  const evalRunId = generateId('evalrun');
1745
- // Use build-time injected eval ID (now properly injected via AST transformation)
1746
- const evalId = evalItem.metadata.id || '';
1768
+
1769
+ // Look up eval metadata from agentuity.metadata.json by agent name and eval name
1770
+ internal.info(
1771
+ `[EVALRUN] Looking up eval metadata: agentName='${agentName}', evalName='${evalName}'`
1772
+ );
1773
+ const evalMeta = getEvalMetadata(agentName, evalName);
1774
+ internal.info(`[EVALRUN] Eval metadata lookup result:`, {
1775
+ found: !!evalMeta,
1776
+ evalId: evalMeta?.evalId,
1777
+ id: evalMeta?.id,
1778
+ filename: evalMeta?.filename,
1779
+ });
1780
+
1781
+ // evalId = deployment-specific ID (evalid_...), evalIdentifier = stable (eval_...)
1782
+ const evalId = evalMeta?.id || '';
1783
+ const evalIdentifier = evalMeta?.evalId || '';
1784
+ internal.info(
1785
+ `[EVALRUN] Resolved evalId='${evalId}', evalIdentifier='${evalIdentifier}'`
1786
+ );
1747
1787
 
1748
1788
  // Log eval metadata using structured logging and tracing
1749
1789
  ctx.logger.debug('Starting eval run with metadata', {
1750
1790
  evalName,
1791
+ agentName,
1751
1792
  evalRunId,
1752
1793
  evalId,
1794
+ evalMetaFromFile: !!evalMeta,
1753
1795
  evalMetadata: evalItem.metadata,
1754
1796
  });
1755
1797
 
@@ -1760,8 +1802,9 @@ export function createAgent<
1760
1802
  'eval.name': evalName,
1761
1803
  'eval.id': evalId,
1762
1804
  'eval.runId': evalRunId,
1763
- 'eval.description': evalItem.metadata.description || '',
1764
- 'eval.filename': evalItem.metadata.filename || '',
1805
+ 'eval.description':
1806
+ evalMeta?.description || evalItem.metadata.description || '',
1807
+ 'eval.filename': evalMeta?.filename || evalItem.metadata.filename || '',
1765
1808
  });
1766
1809
  }
1767
1810
 
@@ -1771,12 +1814,14 @@ export function createAgent<
1771
1814
  const evalRunEventProvider = getEvalRunEventProvider();
1772
1815
 
1773
1816
  // Only send events if we have required context (devmode flag will be set based on devMode)
1774
- const shouldSendEvalRunEvents = orgId && projectId && evalId !== '';
1817
+ const shouldSendEvalRunEvents =
1818
+ orgId && projectId && evalId !== '' && evalIdentifier !== '';
1775
1819
 
1776
1820
  internal.info(`[EVALRUN] Checking conditions for eval '${evalName}':`, {
1777
1821
  orgId: orgId,
1778
1822
  projectId: projectId,
1779
1823
  evalId: evalId,
1824
+ evalIdentifier: evalIdentifier,
1780
1825
  devMode,
1781
1826
  hasEvalRunEventProvider: !!evalRunEventProvider,
1782
1827
  shouldSendEvalRunEvents,
@@ -1787,6 +1832,8 @@ export function createAgent<
1787
1832
  if (!orgId) reasons.push('missing orgId');
1788
1833
  if (!projectId) reasons.push('missing projectId');
1789
1834
  if (!evalId || evalId === '') reasons.push('empty evalId');
1835
+ if (!evalIdentifier || evalIdentifier === '')
1836
+ reasons.push('empty evalIdentifier');
1790
1837
  internal.info(
1791
1838
  `[EVALRUN] Skipping eval run events for '${evalName}': ${reasons.join(', ')}`
1792
1839
  );
@@ -1811,7 +1858,8 @@ export function createAgent<
1811
1858
  const startEvent: EvalRunStartEvent = {
1812
1859
  id: evalRunId,
1813
1860
  sessionId: ctx.sessionId,
1814
- evalId: evalId,
1861
+ evalId: evalId, // deployment-specific ID (evalid_...)
1862
+ evalIdentifier: evalIdentifier, // stable identifier (eval_...)
1815
1863
  orgId: orgId!,
1816
1864
  projectId: projectId!,
1817
1865
  devmode: Boolean(devMode),
@@ -2140,6 +2188,7 @@ const runWithSpan = async <
2140
2188
  '@agentuity/agentInstanceId': agent.metadata.agentId,
2141
2189
  '@agentuity/agentDescription': agent.metadata.description,
2142
2190
  '@agentuity/agentName': agent.metadata.name,
2191
+ '@agentuity/threadId': ctx.var.thread.id,
2143
2192
  });
2144
2193
 
2145
2194
  const spanId = span.spanContext().spanId;
@@ -2244,9 +2293,11 @@ export const createAgentMiddleware = (agentName: AgentName | ''): MiddlewareHand
2244
2293
  const agentKey = toCamelCase(agentName);
2245
2294
  const agent = agentsObj[agentKey];
2246
2295
  const _ctx = privateContext(ctx);
2296
+ // we add both so that you can query by either
2247
2297
  if (agent?.metadata?.id) {
2248
- // we add both so that you can query by either
2249
2298
  _ctx.var.agentIds.add(agent.metadata.id);
2299
+ }
2300
+ if (agent?.metadata?.agentId) {
2250
2301
  _ctx.var.agentIds.add(agent.metadata.agentId);
2251
2302
  }
2252
2303
  }