@agentuity/runtime 0.1.31 → 0.1.33

package/src/middleware.ts

@@ -29,6 +29,7 @@ import * as runtimeConfig from './_config';
 import { getSessionEventProvider } from './_services';
 import { internal } from './logger/internal';
 import { STREAM_DONE_PROMISE_KEY, IS_STREAMING_RESPONSE_KEY } from './handlers/sse';
+import { loadBuildMetadata } from './_metadata';
 
 const SESSION_HEADER = 'x-session-id';
 const THREAD_HEADER = 'x-thread-id';
@@ -302,6 +303,8 @@ export function createOtelMiddleware() {
 					},
 				},
 				async (span): Promise<void> => {
+					// Track request duration from the SDK's perspective
+					const requestStartTime = performance.now();
 					const sctx = span.spanContext();
 					const sessionId = sctx?.traceId ? `sess_${sctx.traceId}` : generateId('sess');
 
@@ -321,6 +324,7 @@ export function createOtelMiddleware() {
 
 					if (projectId) traceState = traceState.set('pid', projectId);
 					if (orgId) traceState = traceState.set('oid', orgId);
+					if (deploymentId) traceState = traceState.set('did', deploymentId);
 					if (isDevMode) traceState = traceState.set('d', '1');
 
 					// Update the active context with the new trace state
@@ -353,8 +357,46 @@ export function createOtelMiddleware() {
 					const sessionEventProvider = getSessionEventProvider();
 					if (sessionEventProvider) {
 						try {
-							// eslint-disable-next-line @typescript-eslint/no-explicit-any
-							const routeId = (c as any).var?.routeId || '';
+							// Look up routeId from build metadata by matching method and path
+							// We need to do this here because the router wrapper hasn't run yet
+							const metadata = loadBuildMetadata();
+							const methodUpper = c.req.method.toUpperCase();
+
+							// Normalize paths: trim trailing slashes for consistent matching
+							const normalizePath = (p: string) => {
+								const decoded = decodeURIComponent(p);
+								return decoded.endsWith('/') && decoded.length > 1
+									? decoded.slice(0, -1)
+									: decoded;
+							};
+							const requestPath = normalizePath(c.req.path);
+
+							// Helper to check if requestPath ends with routePath at a segment boundary
+							// e.g., "/api/translate" matches "/translate" but "/api/translate-v2" does not
+							const matchesAtSegmentBoundary = (reqPath: string, routePath: string) => {
+								if (reqPath === routePath) return true;
+								if (!reqPath.endsWith(routePath)) return false;
+								// Check that the character before the match is a path separator
+								const charBeforeMatch = reqPath[reqPath.length - routePath.length - 1];
+								return charBeforeMatch === '/';
+							};
+
+							// Try matching by exact normalized path first
+							let route = metadata?.routes?.find(
+								(r) =>
+									r.method.toUpperCase() === methodUpper &&
+									normalizePath(r.path) === requestPath
+							);
+							// Fall back to segment-boundary matching (handles /api/translate matching /translate)
+							if (!route) {
+								route = metadata?.routes?.find(
+									(r) =>
+										r.method.toUpperCase() === methodUpper &&
+										matchesAtSegmentBoundary(requestPath, normalizePath(r.path))
+								);
+							}
+							const routeId = route?.id || '';
+
 							await sessionEventProvider.start({
 								id: sessionId,
 								threadId: thread.id,
@@ -418,9 +460,35 @@ export function createOtelMiddleware() {
 						}
 					};
 
+					// Track state for finalization
+					let responseStatus = 200;
+					let errorMessage: string | undefined;
+					let handlerDurationMs = 0;
+					// Track whether span should be ended in finally block (false for streaming - ended in waitUntil)
+					let shouldEndSpanInFinally = true;
+
 					try {
+						internal.info(
+							'[request] %s %s - handler starting (session: %s)',
+							method,
+							url.pathname,
+							sessionId
+						);
+
 						await next();
 
+						// Capture timing immediately after next() returns - this is when the handler completed
+						// This is the HTTP response time we want to report (excludes waitUntil/finalization)
+						handlerDurationMs = performance.now() - requestStartTime;
+
+						internal.info(
+							'[request] %s %s - handler completed in %sms (session: %s)',
+							method,
+							url.pathname,
+							handlerDurationMs.toFixed(2),
+							sessionId
+						);
+
 						// Check if this is a streaming response that needs deferred finalization
 						// eslint-disable-next-line @typescript-eslint/no-explicit-any
 						const streamDone = (c as any).get(STREAM_DONE_PROMISE_KEY) as
@@ -433,67 +501,347 @@ export function createOtelMiddleware() {
 						// or if the response status indicates an error
 						// eslint-disable-next-line @typescript-eslint/no-explicit-any
 						const honoError = (c as any).error as Error | undefined;
-						const responseStatus = c.res?.status ?? 200;
+						responseStatus = c.res?.status ?? 200;
 						const isError = honoError || responseStatus >= 500;
 
+						internal.info(
+							'[request] %s %s - status: %d, streaming: %s, error: %s (session: %s)',
+							method,
+							url.pathname,
+							responseStatus,
+							isStreaming,
+							isError,
+							sessionId
+						);
+
+						if (isError) {
+							// Capture error message for finalization
+							errorMessage = honoError
+								? (honoError.stack ?? honoError.message)
+								: `HTTP ${responseStatus}`;
+							span.setStatus({
+								code: SpanStatusCode.ERROR,
+								message: honoError?.message ?? errorMessage,
+							});
+							if (honoError) {
+								span.recordException(honoError);
+							}
+						} else {
+							span.setStatus({ code: SpanStatusCode.OK });
+						}
+
+						// For streaming responses, defer everything until stream completes
 						if (isStreaming && streamDone) {
-							// Defer session/thread saving until stream completes
-							// This ensures thread state changes made during streaming are persisted
 							internal.info(
-								'[session] deferring session/thread save until streaming completes (session %s)',
+								'[request] %s %s - streaming response, deferring finalization (session: %s)',
+								method,
+								url.pathname,
 								sessionId
 							);
 
+							// For streaming, we end the span inside waitUntil after setting attributes
+							shouldEndSpanInFinally = false;
+
+							// Capture pending promises BEFORE adding finalization waitUntil to avoid deadlock
+							const pendingPromises = handler.getPendingSnapshot();
+							const hasPendingTasks = pendingPromises.length > 0;
+
+							if (hasPendingTasks) {
+								internal.info(
+									'[request] %s %s - %d pending waitUntil tasks to wait for after stream (session: %s)',
+									method,
+									url.pathname,
+									pendingPromises.length,
+									sessionId
+								);
+							}
+
+							// Capture values needed for span attributes (responseStatus already captured above)
+							const capturedResponseStatus = responseStatus;
+							const capturedErrorMessage = errorMessage;
+
+							// Use waitUntil to handle stream completion and finalization
+							// This runs AFTER the response is sent to the client
+							// Note: We intentionally do NOT use noSpan here - the waitUntil span helps
+							// track the streaming finalization work in telemetry
 							handler.waitUntil(async () => {
+								// Track if stream ended with error so we can update finalization status
+								let streamError: unknown = undefined;
+
 								try {
 									await streamDone;
 									internal.info(
-										'[session] stream completed, now saving session/thread (session %s)',
+										'[request] %s %s - stream completed (session: %s)',
+										method,
+										url.pathname,
 										sessionId
 									);
 								} catch (ex) {
-									// Stream ended with an error/abort; still try to persist the latest state
+									streamError = ex;
 									internal.info(
-										'[session] stream ended with error, still saving state: %s',
-										ex
+										'[request] %s %s - stream ended with error: %s (session: %s)',
+										method,
+										url.pathname,
+										ex,
+										sessionId
 									);
 								}
-								await finalizeSession();
-							});
 
-							span.setStatus({ code: SpanStatusCode.OK });
-						} else if (isError) {
-							// Hono caught an error or response is 5xx - report as error
-							const errorMessage = honoError
-								? (honoError.stack ?? honoError.message)
-								: `HTTP ${responseStatus}`;
-							span.setStatus({
-								code: SpanStatusCode.ERROR,
-								message: honoError?.message ?? errorMessage,
+								// Record duration now that stream is complete - set attributes BEFORE ending span
+								const streamDurationMs = performance.now() - requestStartTime;
+								const durationNs = Math.round(streamDurationMs * 1_000_000);
+								internal.info(
+									'[request] %s %s - recording stream duration: %sms (session: %s)',
+									method,
+									url.pathname,
+									streamDurationMs.toFixed(2),
+									sessionId
+								);
+
+								// Determine final status - use stream error if present
+								const finalStatus = streamError ? 500 : capturedResponseStatus;
+								const finalErrorMessage = streamError
+									? streamError instanceof Error
+										? (streamError.stack ?? streamError.message)
+										: String(streamError)
+									: capturedErrorMessage;
+
+								try {
+									// Wait for pending tasks (evals, etc.) captured BEFORE this waitUntil was added
+									if (hasPendingTasks) {
+										internal.info(
+											'[request] %s %s - waiting for %d pending waitUntil tasks (session: %s)',
+											method,
+											url.pathname,
+											pendingPromises.length,
+											sessionId
+										);
+										const logger = c.get('logger');
+										await handler.waitForPromises(pendingPromises, logger, sessionId);
+										internal.info(
+											'[request] %s %s - all waitUntil tasks complete (session: %s)',
+											method,
+											url.pathname,
+											sessionId
+										);
+									}
+
+									// Finalize session after stream completes and evals finish
+									await finalizeSession(
+										finalStatus >= 500 ? finalStatus : undefined,
+										finalErrorMessage
+									);
+									internal.info(
+										'[request] %s %s - stream session finalization complete (session: %s)',
+										method,
+										url.pathname,
+										sessionId
+									);
+								} finally {
+									// Set span attributes and end span AFTER all work is done
+									span.setAttribute('@agentuity/request.duration', durationNs);
+									span.setAttribute('http.status_code', finalStatus);
+
+									// Set span status based on whether there was an error
+									if (streamError) {
+										span.setStatus({
+											code: SpanStatusCode.ERROR,
+											message: finalErrorMessage ?? 'Stream ended with error',
+										});
+										if (streamError instanceof Error) {
+											span.recordException(streamError);
+										}
+									} else {
+										span.setStatus({ code: SpanStatusCode.OK });
+									}
+
+									span.end();
+									internal.info(
+										'[request] %s %s - stream span ended (session: %s)',
+										method,
+										url.pathname,
+										sessionId
+									);
+									// Note: We don't call waitUntilAll() here because this waitUntil callback
+									// IS the final cleanup task. Calling waitUntilAll() would deadlock since
+									// it would wait for this very promise to complete.
+								}
 							});
-							if (honoError) {
-								span.recordException(honoError);
-							}
-							await finalizeSession(responseStatus, errorMessage);
 						} else {
-							// Non-streaming success: save session/thread synchronously
-							await finalizeSession();
-							span.setStatus({ code: SpanStatusCode.OK });
+							// Non-streaming: record duration immediately
+							const durationNs = Math.round(handlerDurationMs * 1_000_000);
+							internal.info(
+								'[request] %s %s - recording duration: %sms (%dns) (session: %s)',
+								method,
+								url.pathname,
+								handlerDurationMs.toFixed(2),
+								durationNs,
+								sessionId
+							);
+							span.setAttribute('@agentuity/request.duration', durationNs);
+							span.setAttribute('http.status_code', responseStatus);
+
+							// Capture pending promises BEFORE adding finalization waitUntil to avoid deadlock.
+							// If we called waitUntilAll inside waitUntil, it would wait for itself.
+							const pendingPromises = handler.getPendingSnapshot();
+							const hasPendingTasks = pendingPromises.length > 0;
+
+							if (hasPendingTasks) {
+								internal.info(
+									'[request] %s %s - %d pending waitUntil tasks to wait for (session: %s)',
+									method,
+									url.pathname,
+									pendingPromises.length,
+									sessionId
+								);
+							}
+
+							// Capture values for use in waitUntil callback
+							const capturedResponseStatus = responseStatus;
+							const capturedErrorMessage = errorMessage;
+
+							// Defer session finalization to run AFTER response is sent
+							// Use noSpan: true since finalizeSession creates its own Session End span
+							handler.waitUntil(
+								async () => {
+									// Wait for the snapshot of pending tasks (evals, etc.) captured BEFORE this waitUntil was added
+									if (hasPendingTasks) {
+										internal.info(
+											'[request] %s %s - waiting for %d pending waitUntil tasks (session: %s)',
+											method,
+											url.pathname,
+											pendingPromises.length,
+											sessionId
+										);
+										const logger = c.get('logger');
+										await handler.waitForPromises(pendingPromises, logger, sessionId);
+										internal.info(
+											'[request] %s %s - all waitUntil tasks complete (session: %s)',
+											method,
+											url.pathname,
+											sessionId
+										);
+									}
+
+									// Finalize session - this is the actual work
+									internal.info(
+										'[request] %s %s - starting session finalization (session: %s)',
+										method,
+										url.pathname,
+										sessionId
+									);
+									try {
+										await finalizeSession(
+											capturedResponseStatus >= 500 ? capturedResponseStatus : undefined,
+											capturedErrorMessage
+										);
+										internal.info(
+											'[request] %s %s - session finalization complete (session: %s)',
+											method,
+											url.pathname,
+											sessionId
+										);
+									} catch (ex) {
+										internal.error(
+											'[request] %s %s - session finalization failed: %s (session: %s)',
+											method,
+											url.pathname,
+											ex,
+											sessionId
+										);
+									}
+									// Note: We don't call waitUntilAll() here because this waitUntil callback
+									// IS the final cleanup task. Calling waitUntilAll() would deadlock since
+									// it would wait for this very promise to complete.
+								},
+								{ noSpan: true }
+							);
 						}
 					} catch (ex) {
+						// Record request metrics even on exceptions (500 status)
+						const exceptionDurationMs = performance.now() - requestStartTime;
+						const durationNs = Math.round(exceptionDurationMs * 1_000_000);
+						internal.info(
+							'[request] %s %s - recording exception duration: %sms (session: %s)',
+							method,
+							url.pathname,
+							exceptionDurationMs.toFixed(2),
+							sessionId
+						);
+						span.setAttribute('@agentuity/request.duration', durationNs);
+						span.setAttribute('http.status_code', 500);
+
 						if (ex instanceof Error) {
 							span.recordException(ex);
 						}
-						const errorMessage = ex instanceof Error ? (ex.stack ?? ex.message) : String(ex);
+						errorMessage = ex instanceof Error ? (ex.stack ?? ex.message) : String(ex);
+						responseStatus = 500;
 						span.setStatus({
 							code: SpanStatusCode.ERROR,
 							message: ex instanceof Error ? ex.message : String(ex),
 						});
 
-						await finalizeSession(500, errorMessage);
+						// Capture error message for use in waitUntil callback
+						const capturedErrorMessage = errorMessage;
+
+						// Capture pending promises BEFORE adding finalization waitUntil to avoid deadlock
+						const pendingPromises = handler.getPendingSnapshot();
+						const hasPendingTasks = pendingPromises.length > 0;
+
+						if (hasPendingTasks) {
+							internal.info(
+								'[request] %s %s - %d pending waitUntil tasks to wait for after error (session: %s)',
+								method,
+								url.pathname,
+								pendingPromises.length,
+								sessionId
+							);
+						}
+
+						// Still defer finalization even on error
+						// Use noSpan: true since finalizeSession creates its own Session End span
+						handler.waitUntil(
+							async () => {
+								// Wait for pending tasks (evals, etc.) captured BEFORE this waitUntil was added
+								if (hasPendingTasks) {
+									internal.info(
+										'[request] %s %s - waiting for %d pending waitUntil tasks (session: %s)',
+										method,
+										url.pathname,
+										pendingPromises.length,
+										sessionId
+									);
+									const logger = c.get('logger');
+									await handler.waitForPromises(pendingPromises, logger, sessionId);
+									internal.info(
+										'[request] %s %s - all waitUntil tasks complete (session: %s)',
+										method,
+										url.pathname,
+										sessionId
+									);
+								}
+
+								try {
+									await finalizeSession(500, capturedErrorMessage);
+								} catch (finalizeEx) {
+									internal.error(
+										'[request] %s %s - error session finalization failed: %s (session: %s)',
+										method,
+										url.pathname,
+										finalizeEx,
+										sessionId
+									);
+								}
+								// Note: We don't call waitUntilAll() here because this waitUntil callback
+								// IS the final cleanup task. Calling waitUntilAll() would deadlock since
+								// it would wait for this very promise to complete.
+							},
+							{ noSpan: true }
+						);
 
 						throw ex;
 					} finally {
+						// Set response headers - this is the only thing that should block the response
 						const headers: Record<string, string> = {};
 						propagation.inject(context.active(), headers);
 						for (const key of Object.keys(headers)) {
@@ -501,7 +849,20 @@ export function createOtelMiddleware() {
 						}
 						const traceId = sctx?.traceId || sessionId.replace(/^sess_/, '');
 						c.header(SESSION_HEADER, `sess_${traceId}`);
-						span.end();
+
+						internal.info(
+							'[request] %s %s - response ready, duration: %sms (session: %s)',
+							method,
+							url.pathname,
+							handlerDurationMs.toFixed(2),
+							sessionId
+						);
+
+						// Only end span here for non-streaming responses
+						// For streaming, span is ended in the waitUntil callback after setting duration attributes
+						if (shouldEndSpanInFinally) {
+							span.end();
+						}
 					}
 				}
 			);

package/src/router.ts

@@ -2,6 +2,7 @@
 import { type Context, Hono, type Schema, type Env as HonoEnv } from 'hono';
 import { returnResponse } from './_util';
 import type { Env } from './app';
+import { loadBuildMetadata } from './_metadata';
 
 // Re-export both Env types
 export type { Env };
@@ -163,8 +164,34 @@ export const createRouter = <E extends Env = Env, S extends Schema = Schema>():
 				return _originalInvoker(path, ...args);
 			}
 
-			// Wrap the handler to add our response conversion
+			// Wrap the handler to add our response conversion and set routeId
 			const wrapper = async (c: Context): Promise<Response> => {
+				// Look up the route ID from build metadata by matching method and path
+				// Try both the registered path and the actual request path (which may include base path)
+				const metadata = loadBuildMetadata();
+				const methodUpper = method.toUpperCase();
+				const requestPath = c.req.routePath || c.req.path;
+
+				// Try matching by registered path first, then by request path, then by path ending
+				let route = metadata?.routes?.find(
+					(r) => r.method.toUpperCase() === methodUpper && r.path === path
+				);
+				if (!route) {
+					route = metadata?.routes?.find(
+						(r) => r.method.toUpperCase() === methodUpper && r.path === requestPath
+					);
+				}
+				if (!route) {
+					// Try matching by path ending (handles /api/translate matching /translate)
+					route = metadata?.routes?.find(
+						(r) => r.method.toUpperCase() === methodUpper && r.path.endsWith(path)
+					);
+				}
+
+				if (route?.id) {
+					(c as any).set('routeId', route.id);
+				}
+
 				let result = handler(c);
 				if (result instanceof Promise) result = await result;
 				// If handler returns a Response, return it unchanged

package/src/services/sandbox/http.ts

@@ -8,6 +8,11 @@ import {
 	sandboxRun,
 	sandboxWriteFiles,
 	sandboxReadFile,
+	snapshotCreate,
+	snapshotGet,
+	snapshotList,
+	snapshotDelete,
+	snapshotTag,
 } from '@agentuity/server';
 import type {
 	SandboxService,
@@ -23,6 +28,11 @@ import type {
 	StreamReader,
 	SandboxStatus,
 	FileToWrite,
+	SnapshotService,
+	SnapshotCreateOptions,
+	SnapshotInfo,
+	SnapshotListParams,
+	SnapshotListResponse,
 } from '@agentuity/core';
 import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
 
@@ -149,13 +159,91 @@ function createSandboxInstance(
 	};
 }
 
+/**
+ * HTTP implementation of the SnapshotService interface
+ */
+class HTTPSnapshotService implements SnapshotService {
+	private client: APIClient;
+
+	constructor(client: APIClient) {
+		this.client = client;
+	}
+
+	async create(sandboxId: string, options?: SnapshotCreateOptions): Promise<SnapshotInfo> {
+		return withSpan(
+			'agentuity.sandbox.snapshot.create',
+			{
+				'sandbox.id': sandboxId,
+				'snapshot.name': options?.name ?? '',
+				'snapshot.tag': options?.tag ?? '',
+			},
+			() =>
+				snapshotCreate(this.client, {
+					sandboxId,
+					name: options?.name,
+					description: options?.description,
+					tag: options?.tag,
+					public: options?.public,
+				})
+		);
+	}
+
+	async get(snapshotId: string): Promise<SnapshotInfo> {
+		return withSpan('agentuity.sandbox.snapshot.get', { 'snapshot.id': snapshotId }, () =>
+			snapshotGet(this.client, { snapshotId })
+		);
+	}
+
+	async list(params?: SnapshotListParams): Promise<SnapshotListResponse> {
+		return withSpan(
+			'agentuity.sandbox.snapshot.list',
+			{
+				'snapshot.sandboxId': params?.sandboxId ?? '',
+				'snapshot.limit': params?.limit ?? 50,
+			},
+			() =>
+				snapshotList(this.client, {
+					sandboxId: params?.sandboxId,
+					limit: params?.limit,
+					offset: params?.offset,
+				})
+		);
+	}
+
+	async delete(snapshotId: string): Promise<void> {
+		return withSpan('agentuity.sandbox.snapshot.delete', { 'snapshot.id': snapshotId }, () =>
+			snapshotDelete(this.client, { snapshotId })
+		);
+	}
+
+	async tag(snapshotId: string, tag: string | null): Promise<SnapshotInfo> {
+		return withSpan(
+			'agentuity.sandbox.snapshot.tag',
+			{
+				'snapshot.id': snapshotId,
+				'snapshot.tag': tag ?? '',
+			},
+			() => snapshotTag(this.client, { snapshotId, tag })
+		);
+	}
+}
+
+/**
+ * HTTP implementation of the SandboxService interface
+ */
 export class HTTPSandboxService implements SandboxService {
 	private client: APIClient;
 	private streamBaseUrl: string;
 
+	/**
+	 * Snapshot management operations
+	 */
+	public readonly snapshot: SnapshotService;
+
 	constructor(client: APIClient, streamBaseUrl: string) {
 		this.client = client;
 		this.streamBaseUrl = streamBaseUrl;
+		this.snapshot = new HTTPSnapshotService(client);
 	}
 
 	async run(options: SandboxRunOptions): Promise<SandboxRunResult> {