@llmops/app 0.3.2 → 0.3.4-beta.1

package/dist/index.cjs

@@ -16432,7 +16432,8 @@ async function handleDirectProviderRequest(c, next, originalBody, providerSlug,
 		...originalBody,
 		model: modelName
 	};
-	delete updatedBody.input;
+	const path$1 = c.req.path;
+	if (path$1.endsWith("/chat/completions") || path$1.endsWith("/completions")) delete updatedBody.input;
 	const newHeaders = new Headers(c.req.raw.headers);
 	newHeaders.set("x-llmops-config", JSON.stringify(portkeyConfig));
 	if (portkeyConfig.default_input_guardrails) newHeaders.set("x-portkey-default-input-guardrails", JSON.stringify(portkeyConfig.default_input_guardrails));
@@ -16474,7 +16475,7 @@ const createGatewayAdapterMiddleware = () => {
 		const method = c.req.method;
 		const contentType = c.req.header("content-type")?.split(";")[0];
 		const isChatRequest = method === "POST" && contentType === "application/json" && (path$1.endsWith("/chat/completions") || path$1.endsWith("/completions"));
-		if (!configId && isChatRequest) {
+		if (!configId && method === "POST" && contentType === "application/json") {
 			try {
 				const body = await c.req.json();
 				const model = body.model;
@@ -16613,12 +16614,49 @@ const createGatewayAdapterMiddleware = () => {
 */
 function createStreamingCostExtractor() {
 	let extractedUsage = null;
+	let extractedHookResults = void 0;
 	let buffer = "";
 	let resolveUsage;
 	const usagePromise = new Promise((resolve) => {
 		resolveUsage = resolve;
 	});
 	const decoder = new TextDecoder();
+	/**
+	* Parse an SSE message and extract usage/hook_results
+	*/
+	function parseSSEMessage(message) {
+		const trimmed = message.trim();
+		if (!trimmed) return;
+		const lines = trimmed.split("\n");
+		let eventType = null;
+		let dataLine = null;
+		for (const line of lines) if (line.startsWith("event:")) eventType = line.slice(6).trim();
+		else if (line.startsWith("data:")) dataLine = line.slice(5).trim();
+		if (dataLine === "[DONE]") return;
+		if (!dataLine) return;
+		try {
+			const parsed = JSON.parse(dataLine);
+			if (eventType === "hook_results" || parsed.hook_results) {
+				const hookData = parsed.hook_results || parsed;
+				if (hookData.before_request_hooks || hookData.after_request_hooks) extractedHookResults = {
+					before_request_hooks: hookData.before_request_hooks,
+					after_request_hooks: hookData.after_request_hooks
+				};
+			}
+			const usageData = parsed;
+			if (usageData.usage) {
+				const promptTokens = usageData.usage.prompt_tokens ?? usageData.usage.input_tokens ?? 0;
+				const completionTokens = usageData.usage.completion_tokens ?? usageData.usage.output_tokens ?? 0;
+				extractedUsage = {
+					promptTokens,
+					completionTokens,
+					totalTokens: usageData.usage.total_tokens ?? promptTokens + completionTokens,
+					cachedTokens: usageData.usage.prompt_tokens_details?.cached_tokens ?? usageData.usage.input_tokens_details?.cached_tokens,
+					hookResults: extractedHookResults
+				};
+			}
+		} catch {}
+	}
 	return {
 		stream: new TransformStream({
 			transform(chunk, controller) {
@@ -16627,39 +16665,17 @@ function createStreamingCostExtractor() {
 				buffer += text;
 				const messages = buffer.split("\n\n");
 				buffer = messages.pop() || "";
-				for (const message of messages) {
-					const trimmed = message.trim();
-					if (!trimmed) continue;
-					if (!trimmed.startsWith("data:")) continue;
-					const jsonPart = trimmed.slice(5).trim();
-					if (jsonPart === "[DONE]") continue;
-					try {
-						const parsed = JSON.parse(jsonPart);
-						if (parsed.usage) extractedUsage = {
-							promptTokens: parsed.usage.prompt_tokens ?? 0,
-							completionTokens: parsed.usage.completion_tokens ?? 0,
-							totalTokens: parsed.usage.total_tokens ?? 0,
-							cachedTokens: parsed.usage.prompt_tokens_details?.cached_tokens
-						};
-					} catch {}
-				}
+				for (const message of messages) parseSSEMessage(message);
 			},
 			flush(controller) {
-				if (buffer.trim()) {
-					const trimmed = buffer.trim();
-					if (trimmed.startsWith("data:")) {
-						const jsonPart = trimmed.slice(5).trim();
-						if (jsonPart !== "[DONE]") try {
-							const parsed = JSON.parse(jsonPart);
-							if (parsed.usage) extractedUsage = {
-								promptTokens: parsed.usage.prompt_tokens ?? 0,
-								completionTokens: parsed.usage.completion_tokens ?? 0,
-								totalTokens: parsed.usage.total_tokens ?? 0,
-								cachedTokens: parsed.usage.prompt_tokens_details?.cached_tokens
-							};
-						} catch {}
-					}
-				}
+				if (buffer.trim()) parseSSEMessage(buffer);
+				if (extractedUsage && extractedHookResults) extractedUsage.hookResults = extractedHookResults;
+				else if (!extractedUsage && extractedHookResults) extractedUsage = {
+					promptTokens: 0,
+					completionTokens: 0,
+					totalTokens: 0,
+					hookResults: extractedHookResults
+				};
 				resolveUsage(extractedUsage);
 			}
 		}),
@@ -16899,6 +16915,47 @@ var PricingProvider = class {
 };
 const pricingProvider = new PricingProvider();
 /**
+* Transform gateway hook results to our schema format for telemetry
+*/
+function transformHookResultsToGuardrailResults(hookResults, wasBlocked) {
+	if (!hookResults) return null;
+	const beforeHooks = hookResults.before_request_hooks || [];
+	const afterHooks = hookResults.after_request_hooks || [];
+	if (beforeHooks.length === 0 && afterHooks.length === 0) return null;
+	const results = [];
+	let totalLatencyMs = 0;
+	for (const hook of beforeHooks) {
+		totalLatencyMs += hook.execution_time;
+		for (const check$1 of hook.checks) results.push({
+			checkId: check$1.id,
+			functionId: check$1.id.split(".")[1] || check$1.id,
+			hookType: "beforeRequestHook",
+			verdict: check$1.verdict,
+			latencyMs: check$1.execution_time
+		});
+	}
+	for (const hook of afterHooks) {
+		totalLatencyMs += hook.execution_time;
+		for (const check$1 of hook.checks) results.push({
+			checkId: check$1.id,
+			functionId: check$1.id.split(".")[1] || check$1.id,
+			hookType: "afterRequestHook",
+			verdict: check$1.verdict,
+			latencyMs: check$1.execution_time
+		});
+	}
+	const anyFailed = results.some((r) => !r.verdict);
+	let action;
+	if (wasBlocked) action = "blocked";
+	else if (anyFailed) action = "logged";
+	else action = "allowed";
+	return {
+		results,
+		action,
+		totalLatencyMs
+	};
+}
+/**
 * Creates cost tracking middleware that logs LLM requests with usage and cost data.
 *
 * Features:
@@ -16913,7 +16970,18 @@ function createCostTrackingMiddleware(config$1 = {}) {
 	return async (c, next) => {
 		if (!enabled) return next();
 		const path$1 = c.req.path;
-		if (!path$1.endsWith("/chat/completions") && !path$1.endsWith("/completions")) return next();
+		if (![
+			"/chat/completions",
+			"/completions",
+			"/responses",
+			"/embeddings",
+			"/images/generations",
+			"/images/edits",
+			"/audio/speech",
+			"/audio/transcriptions",
+			"/audio/translations",
+			"/messages"
+		].some((endpoint) => path$1.endsWith(endpoint) || endpoint === "/responses" && path$1.match(/\/responses\/[^/]+$/))) return next();
 		const requestId = (0, node_crypto.randomUUID)();
 		const startTime = Date.now();
 		c.header("x-llmops-request-id", requestId);
@@ -16976,6 +17044,7 @@ function createCostTrackingMiddleware(config$1 = {}) {
 			const { response: wrappedResponse, usagePromise } = wrapStreamingResponse(response);
 			c.res = wrappedResponse;
 			usagePromise.then(async (usage) => {
+				const guardrailResults = usage?.hookResults ? transformHookResultsToGuardrailResults(usage.hookResults, statusCode === 446) : null;
 				await processUsageAndLog({
 					requestId,
 					provider,
@@ -16994,6 +17063,7 @@ function createCostTrackingMiddleware(config$1 = {}) {
 						totalTokens: usage.totalTokens,
 						cachedTokens: usage.cachedTokens
 					} : null,
+					guardrailResults,
 					tags: customTags,
 					batchWriter,
 					trackErrors,
@@ -17004,14 +17074,24 @@ function createCostTrackingMiddleware(config$1 = {}) {
 			});
 		} else {
 			let usage = null;
+			let guardrailResults = null;
 			try {
 				const responseBody = await response.clone().json();
-				if (responseBody.usage) usage = {
-					promptTokens: responseBody.usage.prompt_tokens || 0,
-					completionTokens: responseBody.usage.completion_tokens || 0,
-					totalTokens: responseBody.usage.total_tokens || 0,
-					cachedTokens: responseBody.usage.prompt_tokens_details?.cached_tokens
-				};
+				if (responseBody.usage) {
+					const promptTokens = responseBody.usage.prompt_tokens ?? responseBody.usage.input_tokens ?? 0;
+					const completionTokens = responseBody.usage.completion_tokens ?? responseBody.usage.output_tokens ?? 0;
+					usage = {
+						promptTokens,
+						completionTokens,
+						totalTokens: responseBody.usage.total_tokens || promptTokens + completionTokens,
+						cachedTokens: responseBody.usage.prompt_tokens_details?.cached_tokens ?? responseBody.usage.input_tokens_details?.cached_tokens
+					};
+				}
+				if (responseBody.hook_results) {
+					const wasBlocked = statusCode === 446;
+					guardrailResults = transformHookResultsToGuardrailResults(responseBody.hook_results, wasBlocked);
+					if (guardrailResults) log(`Extracted guardrail results: ${guardrailResults.results.length} checks, action=${guardrailResults.action}`);
+				}
 			} catch {
 				log("Failed to parse response body for usage");
 			}
@@ -17028,6 +17108,7 @@ function createCostTrackingMiddleware(config$1 = {}) {
 				latencyMs,
 				isStreaming: false,
 				usage,
+				guardrailResults,
 				tags: customTags,
 				batchWriter,
 				trackErrors,
@@ -17040,7 +17121,7 @@ function createCostTrackingMiddleware(config$1 = {}) {
 * Process usage data and log to batch writer
 */
 async function processUsageAndLog(params) {
-	const { requestId, provider, model, configId, variantId, environmentId, providerConfigId, endpoint, statusCode, latencyMs, isStreaming, usage, tags = {}, batchWriter, trackErrors, log } = params;
+	const { requestId, provider, model, configId, variantId, environmentId, providerConfigId, endpoint, statusCode, latencyMs, isStreaming, usage, guardrailResults, tags = {}, batchWriter, trackErrors, log } = params;
 	if (!trackErrors && statusCode >= 400) {
 		log(`Skipping error response (${statusCode})`);
 		return;
@@ -17091,7 +17172,8 @@ async function processUsageAndLog(params) {
 		statusCode,
 		latencyMs,
 		isStreaming,
-		tags
+		tags,
+		guardrailResults: guardrailResults || null
 	};
 	batchWriter.enqueue(requestData);
 	log(`Enqueued request ${requestId} for logging`);

package/dist/index.mjs

@@ -16404,7 +16404,8 @@ async function handleDirectProviderRequest(c, next, originalBody, providerSlug,
 		...originalBody,
 		model: modelName
 	};
-	delete updatedBody.input;
+	const path = c.req.path;
+	if (path.endsWith("/chat/completions") || path.endsWith("/completions")) delete updatedBody.input;
 	const newHeaders = new Headers(c.req.raw.headers);
 	newHeaders.set("x-llmops-config", JSON.stringify(portkeyConfig));
 	if (portkeyConfig.default_input_guardrails) newHeaders.set("x-portkey-default-input-guardrails", JSON.stringify(portkeyConfig.default_input_guardrails));
@@ -16446,7 +16447,7 @@ const createGatewayAdapterMiddleware = () => {
 		const method = c.req.method;
 		const contentType = c.req.header("content-type")?.split(";")[0];
 		const isChatRequest = method === "POST" && contentType === "application/json" && (path.endsWith("/chat/completions") || path.endsWith("/completions"));
-		if (!configId && isChatRequest) {
+		if (!configId && method === "POST" && contentType === "application/json") {
 			try {
 				const body = await c.req.json();
 				const model = body.model;
@@ -16585,12 +16586,49 @@ const createGatewayAdapterMiddleware = () => {
 */
 function createStreamingCostExtractor() {
 	let extractedUsage = null;
+	let extractedHookResults = void 0;
 	let buffer = "";
 	let resolveUsage;
 	const usagePromise = new Promise((resolve) => {
 		resolveUsage = resolve;
 	});
 	const decoder = new TextDecoder();
+	/**
+	* Parse an SSE message and extract usage/hook_results
+	*/
+	function parseSSEMessage(message) {
+		const trimmed = message.trim();
+		if (!trimmed) return;
+		const lines = trimmed.split("\n");
+		let eventType = null;
+		let dataLine = null;
+		for (const line of lines) if (line.startsWith("event:")) eventType = line.slice(6).trim();
+		else if (line.startsWith("data:")) dataLine = line.slice(5).trim();
+		if (dataLine === "[DONE]") return;
+		if (!dataLine) return;
+		try {
+			const parsed = JSON.parse(dataLine);
+			if (eventType === "hook_results" || parsed.hook_results) {
+				const hookData = parsed.hook_results || parsed;
+				if (hookData.before_request_hooks || hookData.after_request_hooks) extractedHookResults = {
+					before_request_hooks: hookData.before_request_hooks,
+					after_request_hooks: hookData.after_request_hooks
+				};
+			}
+			const usageData = parsed;
+			if (usageData.usage) {
+				const promptTokens = usageData.usage.prompt_tokens ?? usageData.usage.input_tokens ?? 0;
+				const completionTokens = usageData.usage.completion_tokens ?? usageData.usage.output_tokens ?? 0;
+				extractedUsage = {
+					promptTokens,
+					completionTokens,
+					totalTokens: usageData.usage.total_tokens ?? promptTokens + completionTokens,
+					cachedTokens: usageData.usage.prompt_tokens_details?.cached_tokens ?? usageData.usage.input_tokens_details?.cached_tokens,
+					hookResults: extractedHookResults
+				};
+			}
+		} catch {}
+	}
 	return {
 		stream: new TransformStream({
 			transform(chunk, controller) {
@@ -16599,39 +16637,17 @@ function createStreamingCostExtractor() {
 				buffer += text;
 				const messages = buffer.split("\n\n");
 				buffer = messages.pop() || "";
-				for (const message of messages) {
-					const trimmed = message.trim();
-					if (!trimmed) continue;
-					if (!trimmed.startsWith("data:")) continue;
-					const jsonPart = trimmed.slice(5).trim();
-					if (jsonPart === "[DONE]") continue;
-					try {
-						const parsed = JSON.parse(jsonPart);
-						if (parsed.usage) extractedUsage = {
-							promptTokens: parsed.usage.prompt_tokens ?? 0,
-							completionTokens: parsed.usage.completion_tokens ?? 0,
-							totalTokens: parsed.usage.total_tokens ?? 0,
-							cachedTokens: parsed.usage.prompt_tokens_details?.cached_tokens
-						};
-					} catch {}
-				}
+				for (const message of messages) parseSSEMessage(message);
 			},
 			flush(controller) {
-				if (buffer.trim()) {
-					const trimmed = buffer.trim();
-					if (trimmed.startsWith("data:")) {
-						const jsonPart = trimmed.slice(5).trim();
-						if (jsonPart !== "[DONE]") try {
-							const parsed = JSON.parse(jsonPart);
-							if (parsed.usage) extractedUsage = {
-								promptTokens: parsed.usage.prompt_tokens ?? 0,
-								completionTokens: parsed.usage.completion_tokens ?? 0,
-								totalTokens: parsed.usage.total_tokens ?? 0,
-								cachedTokens: parsed.usage.prompt_tokens_details?.cached_tokens
-							};
-						} catch {}
-					}
-				}
+				if (buffer.trim()) parseSSEMessage(buffer);
+				if (extractedUsage && extractedHookResults) extractedUsage.hookResults = extractedHookResults;
+				else if (!extractedUsage && extractedHookResults) extractedUsage = {
+					promptTokens: 0,
+					completionTokens: 0,
+					totalTokens: 0,
+					hookResults: extractedHookResults
+				};
 				resolveUsage(extractedUsage);
 			}
 		}),
@@ -16871,6 +16887,47 @@ var PricingProvider = class {
 };
 const pricingProvider = new PricingProvider();
 /**
+* Transform gateway hook results to our schema format for telemetry
+*/
+function transformHookResultsToGuardrailResults(hookResults, wasBlocked) {
+	if (!hookResults) return null;
+	const beforeHooks = hookResults.before_request_hooks || [];
+	const afterHooks = hookResults.after_request_hooks || [];
+	if (beforeHooks.length === 0 && afterHooks.length === 0) return null;
+	const results = [];
+	let totalLatencyMs = 0;
+	for (const hook of beforeHooks) {
+		totalLatencyMs += hook.execution_time;
+		for (const check$1 of hook.checks) results.push({
+			checkId: check$1.id,
+			functionId: check$1.id.split(".")[1] || check$1.id,
+			hookType: "beforeRequestHook",
+			verdict: check$1.verdict,
+			latencyMs: check$1.execution_time
+		});
+	}
+	for (const hook of afterHooks) {
+		totalLatencyMs += hook.execution_time;
+		for (const check$1 of hook.checks) results.push({
+			checkId: check$1.id,
+			functionId: check$1.id.split(".")[1] || check$1.id,
+			hookType: "afterRequestHook",
+			verdict: check$1.verdict,
+			latencyMs: check$1.execution_time
+		});
+	}
+	const anyFailed = results.some((r) => !r.verdict);
+	let action;
+	if (wasBlocked) action = "blocked";
+	else if (anyFailed) action = "logged";
+	else action = "allowed";
+	return {
+		results,
+		action,
+		totalLatencyMs
+	};
+}
+/**
 * Creates cost tracking middleware that logs LLM requests with usage and cost data.
 *
 * Features:
@@ -16885,7 +16942,18 @@ function createCostTrackingMiddleware(config$1 = {}) {
 	return async (c, next) => {
 		if (!enabled) return next();
 		const path = c.req.path;
-		if (!path.endsWith("/chat/completions") && !path.endsWith("/completions")) return next();
+		if (![
+			"/chat/completions",
+			"/completions",
+			"/responses",
+			"/embeddings",
+			"/images/generations",
+			"/images/edits",
+			"/audio/speech",
+			"/audio/transcriptions",
+			"/audio/translations",
+			"/messages"
+		].some((endpoint) => path.endsWith(endpoint) || endpoint === "/responses" && path.match(/\/responses\/[^/]+$/))) return next();
 		const requestId = randomUUID();
 		const startTime = Date.now();
 		c.header("x-llmops-request-id", requestId);
@@ -16948,6 +17016,7 @@ function createCostTrackingMiddleware(config$1 = {}) {
 			const { response: wrappedResponse, usagePromise } = wrapStreamingResponse(response);
 			c.res = wrappedResponse;
 			usagePromise.then(async (usage) => {
+				const guardrailResults = usage?.hookResults ? transformHookResultsToGuardrailResults(usage.hookResults, statusCode === 446) : null;
 				await processUsageAndLog({
 					requestId,
 					provider,
@@ -16966,6 +17035,7 @@ function createCostTrackingMiddleware(config$1 = {}) {
 						totalTokens: usage.totalTokens,
 						cachedTokens: usage.cachedTokens
 					} : null,
+					guardrailResults,
 					tags: customTags,
 					batchWriter,
 					trackErrors,
@@ -16976,14 +17046,24 @@ function createCostTrackingMiddleware(config$1 = {}) {
 			});
 		} else {
 			let usage = null;
+			let guardrailResults = null;
 			try {
 				const responseBody = await response.clone().json();
-				if (responseBody.usage) usage = {
-					promptTokens: responseBody.usage.prompt_tokens || 0,
-					completionTokens: responseBody.usage.completion_tokens || 0,
-					totalTokens: responseBody.usage.total_tokens || 0,
-					cachedTokens: responseBody.usage.prompt_tokens_details?.cached_tokens
-				};
+				if (responseBody.usage) {
+					const promptTokens = responseBody.usage.prompt_tokens ?? responseBody.usage.input_tokens ?? 0;
+					const completionTokens = responseBody.usage.completion_tokens ?? responseBody.usage.output_tokens ?? 0;
+					usage = {
+						promptTokens,
+						completionTokens,
+						totalTokens: responseBody.usage.total_tokens || promptTokens + completionTokens,
+						cachedTokens: responseBody.usage.prompt_tokens_details?.cached_tokens ?? responseBody.usage.input_tokens_details?.cached_tokens
+					};
+				}
+				if (responseBody.hook_results) {
+					const wasBlocked = statusCode === 446;
+					guardrailResults = transformHookResultsToGuardrailResults(responseBody.hook_results, wasBlocked);
+					if (guardrailResults) log(`Extracted guardrail results: ${guardrailResults.results.length} checks, action=${guardrailResults.action}`);
+				}
 			} catch {
 				log("Failed to parse response body for usage");
 			}
@@ -17000,6 +17080,7 @@ function createCostTrackingMiddleware(config$1 = {}) {
 				latencyMs,
 				isStreaming: false,
 				usage,
+				guardrailResults,
 				tags: customTags,
 				batchWriter,
 				trackErrors,
@@ -17012,7 +17093,7 @@ function createCostTrackingMiddleware(config$1 = {}) {
 * Process usage data and log to batch writer
 */
 async function processUsageAndLog(params) {
-	const { requestId, provider, model, configId, variantId, environmentId, providerConfigId, endpoint, statusCode, latencyMs, isStreaming, usage, tags = {}, batchWriter, trackErrors, log } = params;
+	const { requestId, provider, model, configId, variantId, environmentId, providerConfigId, endpoint, statusCode, latencyMs, isStreaming, usage, guardrailResults, tags = {}, batchWriter, trackErrors, log } = params;
 	if (!trackErrors && statusCode >= 400) {
 		log(`Skipping error response (${statusCode})`);
 		return;
@@ -17063,7 +17144,8 @@ async function processUsageAndLog(params) {
 		statusCode,
 		latencyMs,
 		isStreaming,
-		tags
+		tags,
+		guardrailResults: guardrailResults || null
 	};
 	batchWriter.enqueue(requestData);
 	log(`Enqueued request ${requestId} for logging`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llmops/app",
-  "version": "0.3.2",
+  "version": "0.3.4-beta.1",
   "description": "LLMOps application with server and client",
   "type": "module",
   "license": "Apache-2.0",
@@ -67,8 +67,8 @@
     "react-aria-components": "^1.13.0",
     "react-hook-form": "^7.68.0",
     "recharts": "^3.6.0",
-    "@llmops/core": "^0.3.2",
-    "@llmops/gateway": "^0.3.2"
+    "@llmops/core": "^0.3.4-beta.1",
+    "@llmops/gateway": "^0.3.4-beta.1"
   },
   "peerDependencies": {
     "react": "^19.2.1",