@ljoukov/llm 4.0.8 → 4.0.10

package/dist/index.js

@@ -5,7 +5,10 @@ import { randomBytes } from "crypto";
 import {
   FinishReason,
   FunctionCallingConfigMode,
-  ThinkingLevel
+  ThinkingLevel,
+  createFunctionResponsePartFromBase64,
+  createFunctionResponsePartFromUri,
+  createPartFromFunctionResponse
 } from "@google/genai";
 import { zodToJsonSchema } from "@alcyone-labs/zod-to-json-schema";
 import { z as z3 } from "zod";
@@ -2769,6 +2772,33 @@ function ensureTrailingNewline(value) {
 function hasNonEmptyText(value) {
   return typeof value === "string" && value.length > 0;
 }
+function hasLogArtifactValue(value) {
+  if (value === null || value === void 0) {
+    return false;
+  }
+  if (typeof value === "string") {
+    return value.length > 0;
+  }
+  if (Array.isArray(value)) {
+    return value.length > 0;
+  }
+  if (typeof value === "object") {
+    return Object.keys(value).length > 0;
+  }
+  return true;
+}
+function serialiseJsonArtifact(value) {
+  if (!hasLogArtifactValue(value)) {
+    return void 0;
+  }
+  try {
+    return `${JSON.stringify(sanitiseLogValue(value), null, 2)}
+`;
+  } catch {
+    return `${JSON.stringify(String(value), null, 2)}
+`;
+  }
+}
 function redactDataUrlPayload(value) {
   if (!value.toLowerCase().startsWith("data:")) {
     return value;
@@ -3049,7 +3079,9 @@ var AgentLoggingSessionImpl = class {
     const responsePath = path3.join(baseDir, "response.txt");
     const thoughtsPath = path3.join(baseDir, "thoughts.txt");
     const toolCallPath = path3.join(baseDir, "tool_call.txt");
+    const toolCallJsonPath = path3.join(baseDir, "tool_call.json");
     const toolCallResponsePath = path3.join(baseDir, "tool_call_response.txt");
+    const toolCallResponseJsonPath = path3.join(baseDir, "tool_call_response.json");
     const errorPath = path3.join(baseDir, "error.txt");
     const responseMetadataPath = path3.join(baseDir, "response.metadata.json");
     let chain = this.ensureReady.then(async () => {
@@ -3080,6 +3112,10 @@ var AgentLoggingSessionImpl = class {
           "utf8"
         );
       }
+      const toolCallResponseJson = serialiseJsonArtifact(input.toolCallResponsePayload);
+      if (toolCallResponseJson) {
+        await writeFile(toolCallResponseJsonPath, toolCallResponseJson, "utf8");
+      }
     }).catch(() => void 0);
     this.track(chain);
     let closed = false;
@@ -3118,6 +3154,10 @@ var AgentLoggingSessionImpl = class {
           if (hasNonEmptyText(options?.toolCallText)) {
             await writeFile(toolCallPath, ensureTrailingNewline(options.toolCallText), "utf8");
           }
+          const toolCallJson = serialiseJsonArtifact(options?.toolCallPayload);
+          if (toolCallJson) {
+            await writeFile(toolCallJsonPath, toolCallJson, "utf8");
+          }
           await this.writeAttachments(baseDir, options?.attachments);
           const payload = {
             capturedAt: toIsoNow(),
@@ -3147,6 +3187,10 @@ var AgentLoggingSessionImpl = class {
           if (hasNonEmptyText(options?.toolCallText)) {
             await writeFile(toolCallPath, ensureTrailingNewline(options.toolCallText), "utf8");
           }
+          const toolCallJson = serialiseJsonArtifact(options?.toolCallPayload);
+          if (toolCallJson) {
+            await writeFile(toolCallJsonPath, toolCallJson, "utf8");
+          }
           await this.writeAttachments(baseDir, options?.attachments);
           await writeFile(errorPath, ensureTrailingNewline(toErrorMessage(error)), "utf8");
           const payload = {
@@ -4547,6 +4591,150 @@ function toOpenAiToolOutput(value) {
   }
   return mergeToolOutput(value);
 }
+function toGeminiToolOutputItems(value) {
+  if (isLlmToolOutputContentItem(value)) {
+    return [value];
+  }
+  if (Array.isArray(value) && value.every((item) => isLlmToolOutputContentItem(item))) {
+    return value;
+  }
+  return null;
+}
+function inferToolOutputMimeTypeFromFilename(filename) {
+  const normalized = filename?.trim().toLowerCase() ?? "";
+  if (normalized.length === 0) {
+    return void 0;
+  }
+  if (normalized.endsWith(".png")) {
+    return "image/png";
+  }
+  if (normalized.endsWith(".jpg") || normalized.endsWith(".jpeg")) {
+    return "image/jpeg";
+  }
+  if (normalized.endsWith(".webp")) {
+    return "image/webp";
+  }
+  if (normalized.endsWith(".gif")) {
+    return "image/gif";
+  }
+  if (normalized.endsWith(".heic")) {
+    return "image/heic";
+  }
+  if (normalized.endsWith(".heif")) {
+    return "image/heif";
+  }
+  if (normalized.endsWith(".pdf")) {
+    return "application/pdf";
+  }
+  if (normalized.endsWith(".json")) {
+    return "application/json";
+  }
+  if (normalized.endsWith(".md")) {
+    return "text/markdown";
+  }
+  if (normalized.endsWith(".txt")) {
+    return "text/plain";
+  }
+  return void 0;
+}
+function buildGeminiToolOutputMediaPart(item) {
+  if (item.type === "input_image") {
+    const parsed = parseDataUrlPayload(item.image_url);
+    if (!parsed) {
+      return null;
+    }
+    return createFunctionResponsePartFromBase64(parsed.dataBase64, parsed.mimeType);
+  }
+  if (item.type === "input_file") {
+    const dataUrl = typeof item.file_url === "string" ? parseDataUrlPayload(item.file_url) : null;
+    if (dataUrl) {
+      const part = createFunctionResponsePartFromBase64(dataUrl.dataBase64, dataUrl.mimeType);
+      const displayName = item.filename?.trim();
+      if (displayName && part.inlineData) {
+        part.inlineData.displayName = displayName;
+      }
+      return part;
+    }
+    const inferredMimeType = inferToolOutputMimeTypeFromFilename(item.filename);
+    if (typeof item.file_data === "string" && item.file_data.trim().length > 0 && inferredMimeType) {
+      const part = createFunctionResponsePartFromBase64(item.file_data, inferredMimeType);
+      const displayName = item.filename?.trim();
+      if (displayName && part.inlineData) {
+        part.inlineData.displayName = displayName;
+      }
+      return part;
+    }
+    if (typeof item.file_url === "string" && item.file_url.trim().length > 0 && inferredMimeType) {
+      const part = createFunctionResponsePartFromUri(item.file_url, inferredMimeType);
+      const displayName = item.filename?.trim();
+      if (displayName && part.fileData) {
+        part.fileData.displayName = displayName;
+      }
+      return part;
+    }
+  }
+  return null;
+}
+function toGeminiToolOutputPlaceholder(item) {
+  if (item.type === "input_text") {
+    return {
+      type: item.type,
+      text: item.text
+    };
+  }
+  if (item.type === "input_image") {
+    const parsed = parseDataUrlPayload(item.image_url);
+    return {
+      type: item.type,
+      mimeType: parsed?.mimeType ?? void 0,
+      media: "attached-inline-data"
+    };
+  }
+  const dataUrl = typeof item.file_url === "string" ? parseDataUrlPayload(item.file_url) : null;
+  return {
+    type: item.type,
+    filename: item.filename ?? void 0,
+    fileId: item.file_id ?? void 0,
+    mimeType: dataUrl?.mimeType ?? inferToolOutputMimeTypeFromFilename(item.filename) ?? void 0,
+    media: dataUrl || typeof item.file_data === "string" && item.file_data.trim().length > 0 ? "attached-inline-data" : typeof item.file_url === "string" && item.file_url.trim().length > 0 ? "attached-file-data" : void 0
+  };
+}
+function buildGeminiFunctionResponsePart(options) {
+  const outputItems = toGeminiToolOutputItems(options.outputPayload);
+  if (!outputItems) {
+    const responsePayload2 = isPlainRecord(options.outputPayload) ? sanitiseLogValue(options.outputPayload) : { output: sanitiseLogValue(options.outputPayload) };
+    if (options.callId) {
+      return createPartFromFunctionResponse(options.callId, options.toolName, responsePayload2);
+    }
+    return {
+      functionResponse: {
+        name: options.toolName,
+        response: responsePayload2
+      }
+    };
+  }
+  const responseOutput = outputItems.map((item) => toGeminiToolOutputPlaceholder(item));
+  const responseMediaParts = outputItems.flatMap((item) => {
+    const mediaPart = buildGeminiToolOutputMediaPart(item);
+    return mediaPart ? [mediaPart] : [];
+  });
+  const responsePayload = { output: responseOutput };
+  if (options.callId) {
+    return createPartFromFunctionResponse(
+      options.callId,
+      options.toolName,
+      responsePayload,
+      responseMediaParts
+    );
+  }
+  return {
+    functionResponse: {
+      name: options.toolName,
+      response: { output: responseOutput },
+      ...responseMediaParts.length > 0 ? { parts: responseMediaParts } : {}
+    }
+  };
+}
 function parseOpenAiToolArguments(raw) {
   const trimmed = raw.trim();
   if (trimmed.length === 0) {
@@ -5007,7 +5195,7 @@ function resolveAttachmentExtension(mimeType) {
 function buildLoggedAttachmentFilename(prefix, index, mimeType) {
   return `${prefix}-${index.toString()}.${resolveAttachmentExtension(mimeType)}`;
 }
-function decodeDataUrlAttachment(value, options) {
+function parseDataUrlPayload(value) {
   const trimmed = value.trim();
   if (!trimmed.toLowerCase().startsWith("data:")) {
     return null;
@@ -5023,13 +5211,24 @@ function decodeDataUrlAttachment(value, options) {
   try {
     const bytes = isBase64 ? Buffer4.from(payload, "base64") : Buffer4.from(decodeURIComponent(payload), "utf8");
     return {
-      filename: buildLoggedAttachmentFilename(options.prefix, options.index, mimeType),
+      mimeType,
+      dataBase64: bytes.toString("base64"),
       bytes
     };
   } catch {
     return null;
   }
 }
+function decodeDataUrlAttachment(value, options) {
+  const parsed = parseDataUrlPayload(value);
+  if (!parsed) {
+    return null;
+  }
+  return {
+    filename: buildLoggedAttachmentFilename(options.prefix, options.index, parsed.mimeType),
+    bytes: parsed.bytes
+  };
+}
 function collectPayloadAttachments(value, options) {
   if (typeof value === "string") {
     const attachment = decodeDataUrlAttachment(value, {
@@ -5124,6 +5323,10 @@ function collectLoggedAttachmentsFromGeminiParts(parts, prefix) {
   return collectLoggedAttachmentsFromLlmParts(convertGooglePartsToLlmParts(parts), prefix);
 }
 function extractToolCallResponseTextFromOpenAiInput(input) {
+  const responses = extractToolCallResponsePayloadFromOpenAiInput(input);
+  return serialiseLogArtifactText(responses);
+}
+function extractToolCallResponsePayloadFromOpenAiInput(input) {
   if (!Array.isArray(input)) {
     return void 0;
   }
@@ -5140,9 +5343,13 @@ function extractToolCallResponseTextFromOpenAiInput(input) {
       }
     ];
   });
-  return serialiseLogArtifactText(responses);
+  return responses.length > 0 ? responses : void 0;
 }
 function extractToolCallResponseTextFromFireworksMessages(messages) {
+  const responses = extractToolCallResponsePayloadFromFireworksMessages(messages);
+  return serialiseLogArtifactText(responses);
+}
+function extractToolCallResponsePayloadFromFireworksMessages(messages) {
   if (!Array.isArray(messages)) {
     return void 0;
   }
@@ -5157,9 +5364,13 @@ function extractToolCallResponseTextFromFireworksMessages(messages) {
       }
     ];
   });
-  return serialiseLogArtifactText(responses);
+  return responses.length > 0 ? responses : void 0;
 }
 function extractToolCallResponseTextFromGeminiContents(contents) {
+  const responses = extractToolCallResponsePayloadFromGeminiContents(contents);
+  return serialiseLogArtifactText(responses);
+}
+function extractToolCallResponsePayloadFromGeminiContents(contents) {
   if (!Array.isArray(contents)) {
     return void 0;
   }
@@ -5182,40 +5393,36 @@ function extractToolCallResponseTextFromGeminiContents(contents) {
       }
     }
   }
-  return serialiseLogArtifactText(responses);
+  return responses.length > 0 ? responses : void 0;
 }
-function serialiseOpenAiStyleToolCallsForLogging(calls) {
-  return serialiseLogArtifactText(
-    calls.map((call) => {
-      if (call.kind === "custom") {
-        return {
-          kind: call.kind,
-          name: call.name,
-          callId: call.callId,
-          itemId: call.itemId,
-          input: call.input
-        };
-      }
-      const { value, error } = parseOpenAiToolArguments(call.arguments);
+function toLoggedOpenAiStyleToolCalls(calls) {
+  return calls.map((call) => {
+    if (call.kind === "custom") {
       return {
         kind: call.kind,
         name: call.name,
         callId: call.callId,
         itemId: call.itemId,
-        arguments: value,
-        ...error ? { parseError: error, rawArguments: call.arguments } : {}
+        input: call.input
       };
-    })
-  );
+    }
+    const { value, error } = parseOpenAiToolArguments(call.arguments);
+    return {
+      kind: call.kind,
+      name: call.name,
+      callId: call.callId,
+      itemId: call.itemId,
+      arguments: value,
+      ...error ? { parseError: error, rawArguments: call.arguments } : {}
+    };
+  });
 }
-function serialiseGeminiToolCallsForLogging(calls) {
-  return serialiseLogArtifactText(
-    calls.map((call) => ({
-      name: call.name ?? "unknown",
-      callId: typeof call.id === "string" ? call.id : void 0,
-      arguments: sanitiseLogValue(call.args ?? {})
-    }))
-  );
+function toLoggedGeminiToolCalls(calls) {
+  return calls.map((call) => ({
+    name: call.name ?? "unknown",
+    callId: typeof call.id === "string" ? call.id : void 0,
+    arguments: sanitiseLogValue(call.args ?? {})
+  }));
 }
 function startLlmCallLoggerFromContents(options) {
   const session = getCurrentAgentLoggingSession();
@@ -5295,6 +5502,13 @@ function startLlmCallLoggerFromPayload(options) {
   ) : extractToolCallResponseTextFromGeminiContents(
     options.requestPayload.contents
   );
+  const toolCallResponsePayload = options.provider === "openai" || options.provider === "chatgpt" ? extractToolCallResponsePayloadFromOpenAiInput(
+    options.requestPayload.input
+  ) : options.provider === "fireworks" ? extractToolCallResponsePayloadFromFireworksMessages(
+    options.requestPayload.messages
+  ) : extractToolCallResponsePayloadFromGeminiContents(
+    options.requestPayload.contents
+  );
   return session.startLlmCall({
     provider: options.provider,
     modelId: options.modelId,
@@ -5304,7 +5518,8 @@ function startLlmCallLoggerFromPayload(options) {
       ...getCurrentToolCallContext() ? { toolContext: getCurrentToolCallContext() } : {}
     },
     attachments,
-    toolCallResponseText
+    toolCallResponseText,
+    toolCallResponsePayload
   });
 }
 async function runTextCall(params) {
@@ -6135,6 +6350,7 @@ async function runToolLoop(request) {
         let responseText = "";
         let reasoningSummary = "";
         let stepToolCallText;
+        let stepToolCallPayload;
         const stepRequestPayload = {
           model: providerInfo.model,
           input,
@@ -6244,7 +6460,7 @@ async function runToolLoop(request) {
             emitEvent({ type: "usage", usage: usageTokens, costUsd: stepCostUsd, modelVersion });
           }
           const responseToolCalls = extractOpenAiToolCalls(finalResponse.output);
-          stepToolCallText = serialiseOpenAiStyleToolCallsForLogging(
+          stepToolCallPayload = toLoggedOpenAiStyleToolCalls(
             responseToolCalls.map(
               (call) => call.kind === "custom" ? {
                 kind: call.kind,
@@ -6261,6 +6477,7 @@ async function runToolLoop(request) {
               }
             )
           );
+          stepToolCallText = serialiseLogArtifactText(stepToolCallPayload);
           const stepToolCalls = [];
           if (responseToolCalls.length === 0) {
             const steeringInput2 = steeringInternal?.drainPendingContents() ?? [];
@@ -6426,6 +6643,7 @@ async function runToolLoop(request) {
           stepCallLogger?.complete({
             responseText,
             toolCallText: stepToolCallText,
+            toolCallPayload: stepToolCallPayload,
             metadata: {
               provider: "openai",
               model: request.model,
@@ -6446,6 +6664,7 @@ async function runToolLoop(request) {
           stepCallLogger?.fail(error, {
             responseText,
             toolCallText: stepToolCallText,
+            toolCallPayload: stepToolCallPayload,
             metadata: {
               provider: "openai",
               model: request.model,
@@ -6480,6 +6699,7 @@ async function runToolLoop(request) {
         let responseText = "";
         let reasoningSummaryText = "";
         let stepToolCallText;
+        let stepToolCallPayload;
         const markFirstModelEvent = () => {
           if (firstModelEventAtMs === void 0) {
             firstModelEventAtMs = Date.now();
@@ -6548,7 +6768,7 @@ async function runToolLoop(request) {
             stepCallLogger?.appendResponseDelta(responseText);
           }
           const responseToolCalls = response.toolCalls ?? [];
-          stepToolCallText = serialiseOpenAiStyleToolCallsForLogging(
+          stepToolCallPayload = toLoggedOpenAiStyleToolCalls(
             responseToolCalls.map(
               (call) => call.kind === "custom" ? {
                 kind: call.kind,
@@ -6565,6 +6785,7 @@ async function runToolLoop(request) {
               }
             )
           );
+          stepToolCallText = serialiseLogArtifactText(stepToolCallPayload);
           if (responseToolCalls.length === 0) {
             const steeringInput2 = steeringInternal?.drainPendingContents() ?? [];
             const steeringItems2 = steeringInput2.length > 0 ? toChatGptInput(steeringInput2).input : [];
@@ -6737,6 +6958,7 @@ async function runToolLoop(request) {
           stepCallLogger?.complete({
             responseText,
             toolCallText: stepToolCallText,
+            toolCallPayload: stepToolCallPayload,
             metadata: {
               provider: "chatgpt",
               model: request.model,
@@ -6755,6 +6977,7 @@ async function runToolLoop(request) {
           stepCallLogger?.fail(error, {
             responseText,
             toolCallText: stepToolCallText,
+            toolCallPayload: stepToolCallPayload,
             metadata: {
               provider: "chatgpt",
               model: request.model,
@@ -6785,6 +7008,7 @@ async function runToolLoop(request) {
         let responseText = "";
         let blocked = false;
         let stepToolCallText;
+        let stepToolCallPayload;
         const stepRequestPayload = {
           model: providerInfo.model,
           messages,
@@ -6849,7 +7073,7 @@ async function runToolLoop(request) {
             });
           }
           const responseToolCalls = extractFireworksToolCalls(message);
-          stepToolCallText = serialiseOpenAiStyleToolCallsForLogging(
+          stepToolCallPayload = toLoggedOpenAiStyleToolCalls(
             responseToolCalls.map((call) => ({
               kind: "function",
               name: call.name,
@@ -6857,6 +7081,7 @@ async function runToolLoop(request) {
               callId: call.id
             }))
           );
+          stepToolCallText = serialiseLogArtifactText(stepToolCallPayload);
           if (responseToolCalls.length === 0) {
             const steeringInput2 = steeringInternal?.drainPendingContents() ?? [];
             const steeringMessages = steeringInput2.length > 0 ? toFireworksMessages(steeringInput2) : [];
@@ -7009,6 +7234,7 @@ async function runToolLoop(request) {
           stepCallLogger?.complete({
             responseText,
             toolCallText: stepToolCallText,
+            toolCallPayload: stepToolCallPayload,
             metadata: {
               provider: "fireworks",
               model: request.model,
@@ -7037,6 +7263,7 @@ async function runToolLoop(request) {
           stepCallLogger?.fail(error, {
             responseText,
             toolCallText: stepToolCallText,
+            toolCallPayload: stepToolCallPayload,
             metadata: {
               provider: "fireworks",
               model: request.model,
@@ -7065,6 +7292,7 @@ async function runToolLoop(request) {
       let responseText = "";
       let thoughtsText = "";
       let stepToolCallText;
+      let stepToolCallPayload;
       const markFirstModelEvent = () => {
         if (firstModelEventAtMs === void 0) {
           firstModelEventAtMs = Date.now();
@@ -7195,7 +7423,8 @@ async function runToolLoop(request) {
           responseImages: 0
         });
         totalCostUsd += stepCostUsd;
-        stepToolCallText = serialiseGeminiToolCallsForLogging(response.functionCalls);
+        stepToolCallPayload = toLoggedGeminiToolCalls(response.functionCalls);
+        stepToolCallText = serialiseLogArtifactText(stepToolCallPayload);
         if (response.functionCalls.length === 0) {
           const steeringInput2 = steeringInternal?.drainPendingContents() ?? [];
           finalText = responseText;
@@ -7331,14 +7560,13 @@ async function runToolLoop(request) {
             error: result.error,
             durationMs: result.durationMs
           });
-          const responsePayload = isPlainRecord(outputPayload) ? outputPayload : { output: outputPayload };
-          responseParts.push({
-            functionResponse: {
-              name: entry.toolName,
-              response: responsePayload,
-              ...entry.call.id ? { id: entry.call.id } : {}
-            }
-          });
+          responseParts.push(
+            buildGeminiFunctionResponsePart({
+              toolName: entry.toolName,
+              callId: entry.call.id,
+              outputPayload
+            })
+          );
         }
         const stepCompletedAtMs = Date.now();
         const timing = buildStepTiming({
@@ -7364,6 +7592,7 @@ async function runToolLoop(request) {
           responseText,
           attachments: responseOutputAttachments,
           toolCallText: stepToolCallText,
+          toolCallPayload: stepToolCallPayload,
           metadata: {
             provider: "gemini",
             model: request.model,
@@ -7386,6 +7615,7 @@ async function runToolLoop(request) {
         stepCallLogger?.fail(error, {
           responseText,
           toolCallText: stepToolCallText,
+          toolCallPayload: stepToolCallPayload,
           metadata: {
             provider: "gemini",
             model: request.model,