@onlineapps/content-resolver 1.0.2 → 1.1.1

package/README.md

@@ -1,12 +1,21 @@
 # @onlineapps/content-resolver
 
-Automatic conversion between text content and storage references.
+Automatic conversion between text content and storage references with **Content Descriptor Pattern**.
+
+## Overview
+
+ContentResolver provides unified API for working with:
+- **Inline strings** - Small text content (< 16KB)
+- **File references** - Large content stored in MinIO
+- **Binary files** - PDFs, images, etc.
+
+All content is represented as **Content Descriptors** with consistent metadata (filename, content_type, size, fingerprint).
 
 ## Threshold
 
 Default threshold: **16 KB (16384 bytes)**
 
-Content larger than threshold is automatically stored in MinIO and replaced with a reference.
+Content larger than threshold is automatically stored in MinIO and returned as a Descriptor.
 
 ## Usage
 
@@ -29,9 +38,16 @@ const resolver = new ContentResolver({
 const content = await resolver.resolve('minio://workflow/path/to/file.txt');
 // → Returns actual file content
 
-// Store large content as reference
-const result = await resolver.store(largeText, { workflow_id: 'wf-123' });
-// → { value: 'minio://workflow/...', stored: true, size: 50000 }
+// Store large content - returns Content Descriptor
+const descriptor = await resolver.store(largeText, { workflow_id: 'wf-123' }, 'document.html');
+// → {
+//     type: 'file',
+//     storage_ref: 'minio://workflow/...',
+//     filename: 'document.html',
+//     content_type: 'text/html',
+//     size: 50000,
+//     fingerprint: 'sha256...'
+//   }
 ```
 
 ### In Business Service Handler
@@ -71,26 +87,102 @@ exports.processDocument = async (input, context = {}) => {
 If value is a reference (`minio://...`), downloads and returns content.
 Otherwise returns value unchanged.
 
-#### `store(content, context, filename?): Promise<Object>`
-If content size > threshold, stores in MinIO and returns reference.
-Returns `{ value, stored, size, fingerprint? }`.
+#### `store(content, context, filename?, content_type?): Promise<Object>`
+Stores content and returns **Content Descriptor**. If size > threshold, stores in MinIO.
+Returns Descriptor with `type: 'inline'` or `type: 'file'`.
+
+#### `getAsBuffer(value): Promise<Buffer>`
+Unified API to get content as Buffer. Accepts:
+- Plain string → converts to Buffer
+- Storage reference (`minio://...`) → downloads and returns Buffer
+- Content Descriptor → extracts content as Buffer
+
+#### `getAsString(value): Promise<string>`
+Unified API to get content as string. Works with string, reference, or Descriptor.
+
+#### `getMetadata(value): Object`
+Get metadata (filename, content_type, size, fingerprint) from any value type.
+
+#### `createDescriptor(content, options): Promise<Object>`
+Create Content Descriptor from raw content. Automatically decides inline vs file storage.
+
+#### `normalizeToDescriptor(value, options): Promise<Object>`
+Normalize any value (string, reference, Buffer) to Content Descriptor.
 
 #### `resolveInput(input, fields?): Promise<Object>`
-Resolves all reference fields in input object.
+Resolves all reference fields in input object (legacy method).
 
 #### `storeOutput(output, context, fields?): Promise<Object>`
-Stores large content fields as references.
+Stores large content fields as Descriptors (returns Descriptors, not plain strings).
+
+## Content Descriptor Pattern
+
+### Descriptor Structure
+
+```javascript
+// Inline content (small)
+{
+  type: 'inline',
+  content: 'actual text content',
+  encoding: 'utf-8',
+  filename: 'document.txt',
+  content_type: 'text/plain',
+  size: 1234,
+  fingerprint: 'sha256...'
+}
+
+// File content (large or binary)
+{
+  type: 'file',
+  storage_ref: 'minio://workflow/path/to/file',
+  filename: 'invoice.pdf',
+  content_type: 'application/pdf',
+  size: 56607,
+  fingerprint: 'sha256...'
+}
+```
+
+### Usage Example
+
+```javascript
+const resolver = new ContentResolver();
+
+// Work with attachments - unified API
+async function processAttachment(attachment) {
+  // Get content as Buffer (works with string, reference, or Descriptor)
+  const buffer = await resolver.getAsBuffer(attachment.value);
+  
+  // Get metadata
+  const meta = resolver.getMetadata(attachment.value);
+  console.log(`Processing ${meta.filename} (${meta.size} bytes)`);
+  
+  // Process buffer...
+  return processed;
+}
+```
 
 ### Utilities
 
 ```javascript
-const { isReference, parseReference, DEFAULT_THRESHOLD } = require('@onlineapps/content-resolver');
+const { 
+  isReference, 
+  parseReference, 
+  isDescriptor,
+  getContentType,
+  DEFAULT_THRESHOLD 
+} = require('@onlineapps/content-resolver');
 
 isReference('minio://bucket/path');  // true
 isReference('plain text');           // false
 
+isDescriptor({ type: 'file', storage_ref: '...' });  // true
+isDescriptor('plain string');                         // false
+
 parseReference('minio://workflow/content/file.txt');
 // → { bucket: 'workflow', path: 'content/file.txt' }
+
+getContentType('invoice.pdf');  // 'application/pdf'
+getContentType('document.html'); // 'text/html'
 ```
 
 ## Reference Formats

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@onlineapps/content-resolver",
-  "version": "1.0.2",
-  "description": "Automatic conversion between text content and storage references",
+  "version": "1.1.1",
+  "description": "Automatic conversion between text content and storage references with Content Descriptor pattern",
   "main": "src/index.js",
   "scripts": {
     "test": "jest"

package/src/index.js

@@ -2,13 +2,15 @@
  * ContentResolver - Automatic conversion between text content and storage references
  * 
  * Handles transparent conversion:
- * - Large text → stored as file, returns reference
+ * - Large text → stored as file, returns Descriptor
  * - Reference → downloads content, returns text
+ * - Unified Content Descriptor pattern for transparent file/string handling
  * 
  * Used by business services to handle both inline content and file references uniformly.
  */
 
 const StorageConnector = require('@onlineapps/conn-base-storage');
+const crypto = require('crypto');
 
 // Default threshold: 16KB (16384 bytes)
 const DEFAULT_THRESHOLD = 16 * 1024;
@@ -17,6 +19,50 @@ const DEFAULT_THRESHOLD = 16 * 1024;
 const MINIO_REF_PATTERN = /^minio:\/\/([^/]+)\/(.+)$/;
 const INTERNAL_REF_PATTERN = /^internal:\/\/storage\/(.+)$/;
 
+/**
+ * Check if a value is a Content Descriptor
+ * @param {*} value - Value to check
+ * @returns {boolean} True if value is a Descriptor object
+ */
+function isDescriptor(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return false;
+  }
+  return value.type === 'inline' || value.type === 'file';
+}
+
+/**
+ * Get content type from filename or content
+ * @param {string} filename - Filename with extension
+ * @param {string|Buffer} content - Content to analyze
+ * @returns {string} MIME type
+ */
+function getContentType(filename, content) {
+  if (filename) {
+    const ext = filename.split('.').pop()?.toLowerCase();
+    const types = {
+      'html': 'text/html',
+      'htm': 'text/html',
+      'md': 'text/markdown',
+      'txt': 'text/plain',
+      'json': 'application/json',
+      'pdf': 'application/pdf',
+      'png': 'image/png',
+      'jpg': 'image/jpeg',
+      'jpeg': 'image/jpeg',
+      'gif': 'image/gif',
+      'svg': 'image/svg+xml'
+    };
+    if (types[ext]) return types[ext];
+  }
+  
+  // Fallback
+  if (typeof content === 'string' && content.trim().startsWith('<')) {
+    return 'text/html';
+  }
+  return 'text/plain';
+}
+
 /**
  * Check if a value is a storage reference
  * @param {string} value - Value to check
@@ -138,57 +184,32 @@ class ContentResolver {
   }
 
   /**
-   * Store content if above threshold, otherwise return as-is
-   * @param {string} content - Text content to potentially store
+   * Store content if above threshold, otherwise return as Descriptor
+   * @param {string|Buffer} content - Text or binary content to potentially store
    * @param {Object} context - Workflow context for path generation
    * @param {string} [context.workflow_id] - Workflow ID
    * @param {string} [context.step_id] - Step ID
    * @param {string} [filename] - Optional filename hint
-   * @returns {Promise<{ value: string, stored: boolean, size: number }>}
+   * @param {string} [content_type] - Optional MIME type
+   * @returns {Promise<Object>} Content Descriptor
    */
-  async store(content, context = {}, filename = null) {
-    if (!content || typeof content !== 'string') {
-      return { value: content, stored: false, size: 0 };
-    }
-
-    const size = Buffer.byteLength(content, 'utf-8');
-
-    // Below threshold - return as-is
-    if (size <= this.threshold) {
-      return { value: content, stored: false, size };
-    }
-
-    // Above threshold - store in MinIO
-    try {
-      const storage = await this.getStorage();
-      
-      const workflowId = context.workflow_id || 'standalone';
-      const stepId = context.step_id || 'content';
-      const pathPrefix = `content/${workflowId}/${stepId}`;
-      
-      const buffer = Buffer.from(content, 'utf-8');
-      const result = await storage.uploadWithFingerprint(
-        'workflow',
-        buffer,
-        pathPrefix,
-        filename ? filename.split('.').pop() : 'txt'
-      );
-      
-      const ref = `minio://workflow/${result.path}`;
-      this.logger.debug?.(`[ContentResolver] Stored ${size} bytes as ${ref}`);
-      
-      return { 
-        value: ref, 
-        stored: true, 
-        size,
-        fingerprint: result.fingerprint 
+  async store(content, context = {}, filename = null, content_type = null) {
+    if (!content) {
+      return {
+        type: 'inline',
+        content: '',
+        encoding: 'utf-8',
+        filename: filename || 'empty.txt',
+        content_type: content_type || 'text/plain',
+        size: 0
       };
-    } catch (error) {
-      this.logger.error(`[ContentResolver] Failed to store content: ${error.message}`);
-      // Fallback: return original content if storage fails
-      this.logger.warn('[ContentResolver] Falling back to inline content');
-      return { value: content, stored: false, size, warning: error.message };
     }
+
+    return await this.createDescriptor(content, {
+      filename,
+      content_type,
+      context
+    });
   }
 
   /**
@@ -215,11 +236,11 @@ class ContentResolver {
   }
 
   /**
-   * Process an output object - store large string fields as references
+   * Process an output object - store large string fields as Descriptors
    * @param {Object} output - Output object with potential large content
    * @param {Object} context - Workflow context
    * @param {string[]} [fields] - Specific fields to process (default: all string fields)
-   * @returns {Promise<Object>} Output with large content stored as references
+   * @returns {Promise<Object>} Output with large content stored as Descriptors
    */
   async storeOutput(output, context = {}, fields = null) {
     if (!output || typeof output !== 'object') {
@@ -230,13 +251,9 @@ class ContentResolver {
     const fieldsToProcess = fields || Object.keys(output);
 
     for (const field of fieldsToProcess) {
-      if (typeof result[field] === 'string') {
-        const stored = await this.store(result[field], context, field);
-        result[field] = stored.value;
-        if (stored.stored) {
-          result[`${field}_stored`] = true;
-          result[`${field}_size`] = stored.size;
-        }
+      if (typeof result[field] === 'string' || Buffer.isBuffer(result[field])) {
+        const descriptor = await this.store(result[field], context, field);
+        result[field] = descriptor;
       }
     }
 
@@ -258,6 +275,233 @@ class ContentResolver {
   setThreshold(bytes) {
     this.threshold = bytes;
   }
+
+  /**
+   * Download content from storage reference as Buffer
+   * @private
+   * @param {string} ref - Storage reference
+   * @returns {Promise<Buffer>} File content as Buffer
+   */
+  async downloadAsBuffer(ref) {
+    const parsed = parseReference(ref);
+    if (!parsed) {
+      throw new Error(`Invalid storage reference: ${ref}`);
+    }
+
+    const storage = await this.getStorage();
+    const stream = await storage.client.getObject(parsed.bucket, parsed.path);
+    
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+    
+    return Buffer.concat(chunks);
+  }
+
+  /**
+   * Get content as Buffer - unified API for string, reference, or Descriptor
+   * @param {string|Object} value - String, reference, or Content Descriptor
+   * @returns {Promise<Buffer>} Content as Buffer
+   */
+  async getAsBuffer(value) {
+    // Plain string (backward compatibility)
+    if (typeof value === 'string') {
+      if (isReference(value)) {
+        return await this.downloadAsBuffer(value);
+      }
+      return Buffer.from(value, 'utf-8');
+    }
+    
+    // Content Descriptor
+    if (isDescriptor(value)) {
+      if (value.type === 'file') {
+        return await this.downloadAsBuffer(value.storage_ref);
+      }
+      // type === 'inline'
+      const encoding = value.encoding || 'utf-8';
+      return Buffer.from(value.content, encoding);
+    }
+    
+    // Fallback: try to convert to string
+    return Buffer.from(String(value), 'utf-8');
+  }
+
+  /**
+   * Get content as string - unified API for string, reference, or Descriptor
+   * @param {string|Object} value - String, reference, or Content Descriptor
+   * @returns {Promise<string>} Content as string
+   */
+  async getAsString(value) {
+    const buffer = await this.getAsBuffer(value);
+    return buffer.toString('utf-8');
+  }
+
+  /**
+   * Get metadata from value - unified API
+   * @param {string|Object} value - String, reference, or Content Descriptor
+   * @returns {Object} Metadata object with filename, content_type, size, fingerprint
+   */
+  getMetadata(value) {
+    // Plain string
+    if (typeof value === 'string') {
+      return {
+        filename: 'content.txt',
+        content_type: 'text/plain',
+        size: Buffer.byteLength(value, 'utf-8')
+      };
+    }
+    
+    // Content Descriptor
+    if (isDescriptor(value)) {
+      return {
+        filename: value.filename || 'content',
+        content_type: value.content_type || 'text/plain',
+        size: value.size || 0,
+        fingerprint: value.fingerprint || null
+      };
+    }
+    
+    // Fallback
+    const str = String(value);
+    return {
+      filename: 'content.txt',
+      content_type: 'text/plain',
+      size: Buffer.byteLength(str, 'utf-8')
+    };
+  }
+
+  /**
+   * Create Content Descriptor from raw content
+   * @param {string|Buffer} content - Content to create descriptor for
+   * @param {Object} options - Options
+   * @param {string} [options.filename] - Filename hint
+   * @param {string} [options.content_type] - MIME type
+   * @param {Object} [options.context] - Workflow context
+   * @param {boolean} [options.forceFile=false] - Force storage as file even if small
+   * @returns {Promise<Object>} Content Descriptor
+   */
+  async createDescriptor(content, options = {}) {
+    const { filename, content_type, context = {}, forceFile = false } = options;
+    
+    // Convert Buffer to string if needed
+    let contentString;
+    let isBinary = false;
+    if (Buffer.isBuffer(content)) {
+      isBinary = true;
+      // For binary, we'll store it directly
+    } else if (typeof content === 'string') {
+      contentString = content;
+    } else {
+      contentString = String(content);
+    }
+
+    // For binary content, always store as file
+    if (isBinary || forceFile) {
+      const storage = await this.getStorage();
+      const workflowId = context.workflow_id || 'standalone';
+      const stepId = context.step_id || 'content';
+      const pathPrefix = `content/${workflowId}/${stepId}`;
+      
+      const buffer = Buffer.isBuffer(content) ? content : Buffer.from(contentString, 'utf-8');
+      const ext = filename ? filename.split('.').pop() : 'bin';
+      
+      const result = await storage.uploadWithFingerprint(
+        'workflow',
+        buffer,
+        pathPrefix,
+        ext
+      );
+      
+      const finalFilename = filename || `${result.fingerprint.slice(0, 8)}.${ext}`;
+      const finalContentType = content_type || getContentType(finalFilename, buffer);
+      
+      return {
+        type: 'file',
+        storage_ref: `minio://workflow/${result.path}`,
+        filename: finalFilename,
+        content_type: finalContentType,
+        size: buffer.length,
+        fingerprint: result.fingerprint
+      };
+    }
+
+    // For text content, check threshold
+    const size = Buffer.byteLength(contentString, 'utf-8');
+    const finalContentType = content_type || getContentType(filename, contentString);
+    
+    if (size > this.threshold) {
+      // Store as file
+      const storage = await this.getStorage();
+      const workflowId = context.workflow_id || 'standalone';
+      const stepId = context.step_id || 'content';
+      const pathPrefix = `content/${workflowId}/${stepId}`;
+      
+      const buffer = Buffer.from(contentString, 'utf-8');
+      const ext = filename ? filename.split('.').pop() : 'txt';
+      
+      const result = await storage.uploadWithFingerprint(
+        'workflow',
+        buffer,
+        pathPrefix,
+        ext
+      );
+      
+      const finalFilename = filename || `${result.fingerprint.slice(0, 8)}.${ext}`;
+      
+      return {
+        type: 'file',
+        storage_ref: `minio://workflow/${result.path}`,
+        filename: finalFilename,
+        content_type: finalContentType,
+        size: buffer.length,
+        fingerprint: result.fingerprint
+      };
+    }
+
+    // Small content - return as inline
+    const finalFilename = filename || `content_${Date.now()}.txt`;
+    const fingerprint = crypto.createHash('sha256').update(contentString).digest('hex');
+    
+    return {
+      type: 'inline',
+      content: contentString,
+      encoding: 'utf-8',
+      filename: finalFilename,
+      content_type: finalContentType,
+      size: size,
+      fingerprint: fingerprint
+    };
+  }
+
+  /**
+   * Normalize value to Content Descriptor if needed
+   * @param {string|Object} value - String, reference, or Descriptor
+   * @param {Object} options - Options for descriptor creation
+   * @returns {Promise<Object>} Content Descriptor
+   */
+  async normalizeToDescriptor(value, options = {}) {
+    // Already a Descriptor
+    if (isDescriptor(value)) {
+      return value;
+    }
+    
+    // Plain string or reference
+    if (typeof value === 'string') {
+      if (isReference(value)) {
+        // Download and create descriptor
+        const buffer = await this.downloadAsBuffer(value);
+        const parsed = parseReference(value);
+        const filename = parsed ? parsed.path.split('/').pop() : 'file';
+        return await this.createDescriptor(buffer, { ...options, filename, forceFile: true });
+      }
+      // Plain string - create descriptor
+      return await this.createDescriptor(value, options);
+    }
+    
+    // Fallback
+    return await this.createDescriptor(String(value), options);
+  }
 }
 
 // Export class and utilities
@@ -265,5 +509,7 @@ module.exports = ContentResolver;
 module.exports.ContentResolver = ContentResolver;
 module.exports.isReference = isReference;
 module.exports.parseReference = parseReference;
+module.exports.isDescriptor = isDescriptor;
+module.exports.getContentType = getContentType;
 module.exports.DEFAULT_THRESHOLD = DEFAULT_THRESHOLD;