@dexto/core 1.1.10 → 1.2.0

package/dist/context/manager.js

@@ -1,7 +1,14 @@
+import "../chunk-C6A6W6XS.js";
 import { MiddleRemovalStrategy } from "./compression/middle-removal.js";
 import { OldestRemovalStrategy } from "./compression/oldest-removal.js";
 import { logger } from "../logger/index.js";
-import { countMessagesTokens, sanitizeToolResultToContent } from "./utils.js";
+import { eventBus } from "../events/index.js";
+import {
+  countMessagesTokens,
+  sanitizeToolResult,
+  expandBlobReferences,
+  isLikelyBase64String
+} from "./utils.js";
 import { ContextError } from "./errors.js";
 class ContextManager {
   /**
@@ -9,9 +16,9 @@ class ContextManager {
    */
   llmConfig;
   /**
-   * PromptManager used to generate/manage the system prompt
+   * SystemPromptManager used to generate/manage the system prompt
    */
-  promptManager;
+  systemPromptManager;
   /**
    * Formatter used to convert internal messages to LLM-specific format
    */
@@ -43,29 +50,96 @@ class ContextManager {
   compressionStrategies;
   historyProvider;
   sessionId;
+  /**
+   * ResourceManager for resolving blob references in message content.
+   * Blob references like @blob:abc123 are resolved to actual data
+   * before passing messages to the LLM formatter.
+   */
+  resourceManager;
+  /**
+   * Get the ResourceManager instance
+   */
+  getResourceManager() {
+    return this.resourceManager;
+  }
+  /**
+   * Process user input data - store as blob if large, otherwise return as-is.
+   * Returns either the original data or a blob reference (@blob:id).
+   */
+  async processUserInput(data, metadata) {
+    const blobService = this.resourceManager.getBlobStore();
+    let shouldStoreAsBlob = false;
+    let estimatedSize = 0;
+    if (typeof data === "string") {
+      if (data.startsWith("data:")) {
+        const commaIndex = data.indexOf(",");
+        if (commaIndex !== -1) {
+          const base64Data = data.substring(commaIndex + 1);
+          estimatedSize = Math.floor(base64Data.length * 3 / 4);
+        }
+      } else if (data.length > 100 && data.match(/^[A-Za-z0-9+/=]+$/)) {
+        estimatedSize = Math.floor(data.length * 3 / 4);
+      } else {
+        estimatedSize = Buffer.byteLength(data, "utf8");
+      }
+    } else if (data instanceof Buffer || data instanceof Uint8Array) {
+      estimatedSize = data.length;
+    } else if (data instanceof ArrayBuffer) {
+      estimatedSize = data.byteLength;
+    } else if (data instanceof URL) {
+      return data;
+    }
+    const isLikelyBinary = metadata.mimeType.startsWith("image/") || metadata.mimeType.startsWith("audio/") || metadata.mimeType.startsWith("video/") || metadata.mimeType === "application/pdf";
+    shouldStoreAsBlob = isLikelyBinary || estimatedSize > 5 * 1024;
+    if (shouldStoreAsBlob) {
+      try {
+        const blobInput = typeof data === "string" && !data.startsWith("data:") && !isLikelyBase64String(data) && !isLikelyBinary ? Buffer.from(data, "utf-8") : data;
+        const blobRef = await blobService.store(blobInput, {
+          mimeType: metadata.mimeType,
+          originalName: metadata.originalName,
+          source: metadata.source || "user"
+        });
+        logger.info(
+          `Stored user input as blob: ${blobRef.uri} (${estimatedSize} bytes, ${metadata.mimeType})`
+        );
+        eventBus.emit("dexto:resourceCacheInvalidated", {
+          resourceUri: blobRef.uri,
+          serverName: "internal",
+          action: "blob_stored"
+        });
+        return `@${blobRef.uri}`;
+      } catch (error) {
+        logger.warn(`Failed to store user input as blob: ${String(error)}`);
+        return data;
+      }
+    }
+    return data;
+  }
   /**
    * Creates a new ContextManager instance
    * @param llmConfig The validated LLM configuration.
    * @param formatter Formatter implementation for the target LLM provider
-   * @param promptManager PromptManager instance for the conversation
+   * @param systemPromptManager SystemPromptManager instance for the conversation
    * @param maxInputTokens Maximum token limit for the conversation history. Triggers compression if exceeded and a tokenizer is provided.
    * @param tokenizer Tokenizer implementation used for counting tokens and enabling compression.
    * @param historyProvider Session-scoped ConversationHistoryProvider instance for managing conversation history
    * @param sessionId Unique identifier for the conversation session (readonly, for debugging)
    * @param compressionStrategies Optional array of compression strategies to apply when token limits are exceeded
+   * @param resourceManager Optional ResourceManager for resolving blob references in messages
    */
-  constructor(llmConfig, formatter, promptManager, maxInputTokens, tokenizer, historyProvider, sessionId, compressionStrategies = [
+  constructor(llmConfig, formatter, systemPromptManager, maxInputTokens, tokenizer, historyProvider, sessionId, resourceManager, compressionStrategies = [
     new MiddleRemovalStrategy(),
     new OldestRemovalStrategy()
   ]) {
     this.llmConfig = llmConfig;
     this.formatter = formatter;
-    this.promptManager = promptManager;
+    this.systemPromptManager = systemPromptManager;
     this.maxInputTokens = maxInputTokens;
     this.tokenizer = tokenizer;
     this.historyProvider = historyProvider;
     this.sessionId = sessionId;
     this.compressionStrategies = compressionStrategies;
+    this.resourceManager = resourceManager;
     logger.debug(
       `ContextManager: Initialized for session ${sessionId} - history will be managed by ${historyProvider.constructor.name}`
     );
@@ -157,10 +231,10 @@ class ContextManager {
     return estimatedTotal > compressionTrigger;
   }
   /**
-   * Assembles and returns the current system prompt by invoking the PromptManager.
+   * Assembles and returns the current system prompt by invoking the SystemPromptManager.
    */
   async getSystemPrompt(context) {
-    const prompt = await this.promptManager.build(context);
+    const prompt = await this.systemPromptManager.build(context);
     logger.debug(`[SystemPrompt] Built system prompt:
 ${prompt}`);
     return prompt;
@@ -248,16 +322,28 @@ ${prompt}`);
       messageParts.push({ type: "text", text: finalTextContent });
     }
     if (imageData) {
+      const processedImage = await this.processUserInput(imageData.image, {
+        mimeType: imageData.mimeType || "image/jpeg",
+        source: "user"
+      });
       messageParts.push({
         type: "image",
-        image: imageData.image,
+        image: processedImage,
         mimeType: imageData.mimeType || "image/jpeg"
       });
     }
     if (fileData) {
+      const metadata = {
+        mimeType: fileData.mimeType,
+        source: "user"
+      };
+      if (fileData.filename) {
+        metadata.originalName = fileData.filename;
+      }
+      const processedData = await this.processUserInput(fileData.data, metadata);
       messageParts.push({
         type: "file",
-        data: fileData.data,
+        data: processedData,
         mimeType: fileData.mimeType,
         ...fileData.filename && { filename: fileData.filename }
       });
@@ -299,24 +385,31 @@ ${prompt}`);
    * @param result The result returned by the tool
    * @throws Error if required parameters are missing
    */
-  async addToolResult(toolCallId, name, result) {
+  async addToolResult(toolCallId, name, result, options) {
     if (!toolCallId || !name) {
       throw ContextError.toolCallIdNameRequired();
     }
-    let content;
-    content = sanitizeToolResultToContent(result);
-    if (typeof content === "string") {
-      const preview = content.slice(0, 200);
-      logger.debug(
-        `ContextManager: Storing tool result (text) for ${name} (len=${content.length}): ${preview}${content.length > 200 ? "..." : ""}`
-      );
-    } else if (Array.isArray(content)) {
-      const summary = content.map(
-        (p) => p.type === "text" ? `text(${p.text.length})` : p.type === "image" ? `image(${p.mimeType || "image"})` : `file(${p.mimeType || "file"})`
-      ).join(", ");
-      logger.debug(`ContextManager: Storing tool result (parts) for ${name}: [${summary}]`);
+    const blobService = this.resourceManager.getBlobStore();
+    const sanitizeOptions = {
+      blobStore: blobService,
+      toolName: name,
+      toolCallId
+    };
+    if (options?.success !== void 0) {
+      sanitizeOptions.success = options.success;
     }
-    await this.addMessage({ role: "tool", content, toolCallId, name });
+    const sanitized = await sanitizeToolResult(result, sanitizeOptions);
+    const summary = sanitized.content.map(
+      (p) => p.type === "text" ? `text(${p.text.length})` : p.type === "image" ? `image(${p.mimeType || "image"})` : `file(${p.mimeType || "file"})`
+    ).join(", ");
+    logger.debug(`ContextManager: Storing tool result (parts) for ${name}: [${summary}]`);
+    await this.addMessage({
+      role: "tool",
+      content: sanitized.content,
+      toolCallId,
+      name
+    });
+    return sanitized;
   }
   /**
    * Sets the system prompt for the conversation
@@ -339,7 +432,42 @@ ${prompt}`);
    * @throws Error if formatting or compression fails critically
    */
   async getFormattedMessages(contributorContext, llmContext, systemPrompt, history) {
-    const messageHistory = history ?? await this.historyProvider.getHistory();
+    let messageHistory = history ?? await this.historyProvider.getHistory();
+    let allowedMediaTypes = this.llmConfig.allowedMediaTypes;
+    if (!allowedMediaTypes) {
+      try {
+        const { getSupportedFileTypesForModel } = await import("../llm/registry.js");
+        const { fileTypesToMimePatterns } = await import("./utils.js");
+        const supportedFileTypes = getSupportedFileTypesForModel(
+          llmContext.provider,
+          llmContext.model
+        );
+        allowedMediaTypes = fileTypesToMimePatterns(supportedFileTypes);
+        logger.debug(
+          `Using model capabilities for media filtering: ${allowedMediaTypes.join(", ")}`
+        );
+      } catch (error) {
+        logger.warn(
+          `Could not determine model capabilities, allowing all media types: ${String(error)}`
+        );
+        allowedMediaTypes = void 0;
+      }
+    } else {
+      logger.debug(
+        `Using user-configured allowedMediaTypes: ${allowedMediaTypes.join(", ")}`
+      );
+    }
+    logger.debug("Resolving blob references in message history before formatting");
+    messageHistory = await Promise.all(
+      messageHistory.map(async (message) => {
+        const expandedContent = await expandBlobReferences(
+          message.content,
+          this.resourceManager,
+          allowedMediaTypes
+        );
+        return { ...message, content: expandedContent };
+      })
+    );
     const prompt = systemPrompt ?? await this.getSystemPrompt(contributorContext);
     return this.formatter.format([...messageHistory], llmContext, prompt);
   }

package/dist/context/media-helpers.cjs

@@ -0,0 +1,40 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var media_helpers_exports = {};
+__export(media_helpers_exports, {
+  getFileMediaKind: () => getFileMediaKind,
+  getResourceKind: () => getResourceKind
+});
+module.exports = __toCommonJS(media_helpers_exports);
+function getFileMediaKind(mimeType) {
+  if (mimeType?.startsWith("audio/")) return "audio";
+  if (mimeType?.startsWith("video/")) return "video";
+  return "binary";
+}
+function getResourceKind(mimeType) {
+  if (mimeType?.startsWith("image/")) return "image";
+  if (mimeType?.startsWith("audio/")) return "audio";
+  if (mimeType?.startsWith("video/")) return "video";
+  return "binary";
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getFileMediaKind,
+  getResourceKind
+});

package/dist/context/media-helpers.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Browser-safe media kind helpers.
+ * These functions have no dependencies and can be safely imported in browser environments.
+ */
+/**
+ * Derive file media kind from MIME type.
+ * This is the canonical way to determine media kind - use this instead of storing redundant fields.
+ */
+export declare function getFileMediaKind(mimeType: string | undefined): 'audio' | 'video' | 'binary';
+/**
+ * Derive resource kind from MIME type (includes images).
+ * Use this to determine the kind of resource for display/rendering purposes.
+ */
+export declare function getResourceKind(mimeType: string | undefined): 'image' | 'audio' | 'video' | 'binary';
+//# sourceMappingURL=media-helpers.d.ts.map

package/dist/context/media-helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"media-helpers.d.ts","sourceRoot":"","sources":["../../src/context/media-helpers.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,CAI3F;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,GAC7B,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,CAKxC"}

package/dist/context/media-helpers.js

@@ -0,0 +1,16 @@
+import "../chunk-C6A6W6XS.js";
+function getFileMediaKind(mimeType) {
+  if (mimeType?.startsWith("audio/")) return "audio";
+  if (mimeType?.startsWith("video/")) return "video";
+  return "binary";
+}
+function getResourceKind(mimeType) {
+  if (mimeType?.startsWith("image/")) return "image";
+  if (mimeType?.startsWith("audio/")) return "audio";
+  if (mimeType?.startsWith("video/")) return "video";
+  return "binary";
+}
+export {
+  getFileMediaKind,
+  getResourceKind
+};

package/dist/context/types.d.ts

@@ -1,29 +1,47 @@
-import { LLMProvider, LLMRouter } from '../llm/types.js';
-
+import type { LLMRouter, LLMProvider } from '../llm/types.js';
 /**
  * Internal representation of a message in a conversation.
  * Standardizes message format across different LLM providers.
  */
-interface ImageData {
+export interface ImageData {
     image: string | Uint8Array | Buffer | ArrayBuffer | URL;
     mimeType?: string;
 }
-interface FileData {
+export interface FileData {
     data: string | Uint8Array | Buffer | ArrayBuffer | URL;
     mimeType: string;
     filename?: string;
 }
-interface TextPart {
+export interface TextPart {
     type: 'text';
     text: string;
 }
-interface ImagePart extends ImageData {
+export interface ImagePart extends ImageData {
     type: 'image';
 }
-interface FilePart extends FileData {
+export interface FilePart extends FileData {
     type: 'file';
 }
-interface InternalMessage {
+export interface SanitizedToolResult {
+    /** Ordered content parts ready for rendering or provider formatting */
+    content: Array<TextPart | ImagePart | FilePart>;
+    /**
+     * Resource references created during sanitization (e.g. blob store URIs).
+     * Consumers can dereference these via ResourceManager APIs.
+     */
+    resources?: Array<{
+        uri: string;
+        kind: 'image' | 'audio' | 'video' | 'binary';
+        mimeType: string;
+        filename?: string;
+    }>;
+    meta: {
+        toolName: string;
+        toolCallId: string;
+        success?: boolean;
+    };
+}
+export interface InternalMessage {
     /**
      * The role of the entity sending the message.
      * - 'system': System instructions or context
@@ -109,5 +127,4 @@ interface InternalMessage {
      */
     name?: string;
 }
-
-export type { FileData, FilePart, ImageData, ImagePart, InternalMessage, TextPart };
+//# sourceMappingURL=types.d.ts.map

package/dist/context/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/context/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9D;;;GAGG;AACH,MAAM,WAAW,SAAS;IACtB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,WAAW,GAAG,GAAG,CAAC;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACrB,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,WAAW,GAAG,GAAG,CAAC;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,SAAU,SAAQ,SAAS;IACxC,IAAI,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,QAAS,SAAQ,QAAQ;IACtC,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IAChC,uEAAuE;IACvE,OAAO,EAAE,KAAK,CAAC,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAC,CAAC;IAChD;;;OAGG;IACH,SAAS,CAAC,EAAE,KAAK,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,CAAC;QAC7C,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;IACH,IAAI,EAAE;QACF,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;CACL;AAED,MAAM,WAAW,eAAe;IAC5B;;;;;;OAMG;IACH,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC;IAE/C;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;OAKG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,KAAK,CAAC,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAC,CAAC;IAEhE;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,CAAC,EAAE;QACT,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;IAEF;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,WAAW,CAAC;IAEvB;;;OAGG;IACH,MAAM,CAAC,EAAE,SAAS,CAAC;IAEnB;;;OAGG;IACH,SAAS,CAAC,EAAE,KAAK,CAAC;QACd;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,IAAI,EAAE,UAAU,CAAC;QAEjB;;WAEG;QACH,QAAQ,EAAE;YACN;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;YAEb;;eAEG;YACH,SAAS,EAAE,MAAM,CAAC;SACrB,CAAC;KACL,CAAC,CAAC;IAEH;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB"}