@onlineapps/content-resolver 1.0.1

package/README.md

@@ -0,0 +1,101 @@
+# @onlineapps/content-resolver
+
+Automatic conversion between text content and storage references.
+
+## Threshold
+
+Default threshold: **16 KB (16384 bytes)**
+
+Content larger than threshold is automatically stored in MinIO and replaced with a reference.
+
+## Usage
+
+### Basic Usage
+
+```javascript
+const ContentResolver = require('@onlineapps/content-resolver');
+
+const resolver = new ContentResolver({
+  threshold: 16 * 1024,  // 16KB (default)
+  storage: {
+    endPoint: 'api_shared_storage',
+    port: 9000,
+    accessKey: 'minioadmin',
+    secretKey: 'minioadmin'
+  }
+});
+
+// Resolve reference to content
+const content = await resolver.resolve('minio://workflow/path/to/file.txt');
+// → Returns actual file content
+
+// Store large content as reference
+const result = await resolver.store(largeText, { workflow_id: 'wf-123' });
+// → { value: 'minio://workflow/...', stored: true, size: 50000 }
+```
+
+### In Business Service Handler
+
+```javascript
+const ContentResolver = require('@onlineapps/content-resolver');
+
+exports.processDocument = async (input, context = {}) => {
+  const resolver = new ContentResolver();
+  
+  // Input can be either text or reference - resolve transparently
+  const resolvedInput = await resolver.resolveInput(input, ['content', 'markdown']);
+  
+  // Now resolvedInput.content and resolvedInput.markdown are always text
+  const result = processContent(resolvedInput.content);
+  
+  // Store large output as reference
+  const output = await resolver.storeOutput({ result }, context, ['result']);
+  
+  return output;
+};
+```
+
+## API
+
+### `new ContentResolver(options)`
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `threshold` | number | 16384 | Size threshold in bytes |
+| `storage` | Object | env-based | Storage connector config |
+| `logger` | Object | console | Logger instance |
+
+### Methods
+
+#### `resolve(value): Promise<string>`
+If value is a reference (`minio://...`), downloads and returns content.
+Otherwise returns value unchanged.
+
+#### `store(content, context, filename?): Promise<Object>`
+If content size > threshold, stores in MinIO and returns reference.
+Returns `{ value, stored, size, fingerprint? }`.
+
+#### `resolveInput(input, fields?): Promise<Object>`
+Resolves all reference fields in input object.
+
+#### `storeOutput(output, context, fields?): Promise<Object>`
+Stores large content fields as references.
+
+### Utilities
+
+```javascript
+const { isReference, parseReference, DEFAULT_THRESHOLD } = require('@onlineapps/content-resolver');
+
+isReference('minio://bucket/path');  // true
+isReference('plain text');           // false
+
+parseReference('minio://workflow/content/file.txt');
+// → { bucket: 'workflow', path: 'content/file.txt' }
+```
+
+## Reference Formats
+
+Supported formats:
+- `minio://bucket/path/to/file`
+- `internal://storage/path/to/file` (uses default bucket)
+

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@onlineapps/content-resolver",
+  "version": "1.0.1",
+  "description": "Automatic conversion between text content and storage references",
+  "main": "src/index.js",
+  "scripts": {
+    "test": "jest"
+  },
+  "keywords": [
+    "content",
+    "storage",
+    "minio",
+    "reference",
+    "resolver"
+  ],
+  "author": "OnlineApps",
+  "license": "ISC",
+  "dependencies": {
+    "@onlineapps/conn-base-storage": "^1.0.0"
+  },
+  "peerDependencies": {
+    "@onlineapps/conn-base-storage": "^1.0.0"
+  }
+}

package/src/index.js

@@ -0,0 +1,269 @@
+/**
+ * ContentResolver - Automatic conversion between text content and storage references
+ * 
+ * Handles transparent conversion:
+ * - Large text → stored as file, returns reference
+ * - Reference → downloads content, returns text
+ * 
+ * Used by business services to handle both inline content and file references uniformly.
+ */
+
+const StorageConnector = require('@onlineapps/conn-base-storage');
+
+// Default threshold: 16KB (16384 bytes)
+const DEFAULT_THRESHOLD = 16 * 1024;
+
+// Reference patterns
+const MINIO_REF_PATTERN = /^minio:\/\/([^/]+)\/(.+)$/;
+const INTERNAL_REF_PATTERN = /^internal:\/\/storage\/(.+)$/;
+
+/**
+ * Check if a value is a storage reference
+ * @param {string} value - Value to check
+ * @returns {boolean} True if value is a storage reference
+ */
+function isReference(value) {
+  if (typeof value !== 'string') return false;
+  return MINIO_REF_PATTERN.test(value) || INTERNAL_REF_PATTERN.test(value);
+}
+
+/**
+ * Parse a storage reference into bucket and path
+ * @param {string} ref - Storage reference (minio://bucket/path or internal://storage/path)
+ * @returns {{ bucket: string, path: string } | null}
+ */
+function parseReference(ref) {
+  const minioMatch = ref.match(MINIO_REF_PATTERN);
+  if (minioMatch) {
+    return { bucket: minioMatch[1], path: minioMatch[2] };
+  }
+  
+  const internalMatch = ref.match(INTERNAL_REF_PATTERN);
+  if (internalMatch) {
+    // Internal references use default bucket
+    return { bucket: 'workflow', path: internalMatch[1] };
+  }
+  
+  return null;
+}
+
+/**
+ * ContentResolver class
+ */
+class ContentResolver {
+  /**
+   * Create a ContentResolver
+   * @param {Object} options - Configuration options
+   * @param {number} [options.threshold=16384] - Size threshold in bytes (default 16KB)
+   * @param {Object} [options.storage] - Storage connector config or instance
+   * @param {Object} [options.logger] - Logger instance
+   */
+  constructor(options = {}) {
+    this.threshold = options.threshold || DEFAULT_THRESHOLD;
+    this.logger = options.logger || console;
+    
+    // Storage can be passed as instance or config
+    if (options.storage instanceof StorageConnector) {
+      this.storage = options.storage;
+    } else {
+      this.storageConfig = options.storage || {
+        endPoint: process.env.MINIO_HOST || 'api_shared_storage',
+        port: parseInt(process.env.MINIO_PORT || '9000'),
+        useSSL: false,
+        accessKey: process.env.MINIO_ACCESS_KEY || 'minioadmin',
+        secretKey: process.env.MINIO_SECRET_KEY || 'minioadmin',
+        defaultBucket: 'workflow'
+      };
+      this.storage = null;
+    }
+  }
+
+  /**
+   * Get or create storage connector
+   * @returns {Promise<StorageConnector>}
+   */
+  async getStorage() {
+    if (!this.storage) {
+      // Remove protocol from endpoint if present
+      const config = { ...this.storageConfig };
+      if (config.endPoint) {
+        config.endPoint = config.endPoint.replace(/^https?:\/\//, '');
+      }
+      this.storage = new StorageConnector(config);
+      await this.storage.initialize();
+    }
+    return this.storage;
+  }
+
+  /**
+   * Resolve a value - if it's a reference, download content; if it's text, return as-is
+   * @param {string} value - Text content or storage reference
+   * @returns {Promise<string>} Resolved text content
+   */
+  async resolve(value) {
+    if (!value || typeof value !== 'string') {
+      return value;
+    }
+
+    // Check if it's a reference
+    if (!isReference(value)) {
+      return value; // Already text content
+    }
+
+    // Parse reference
+    const parsed = parseReference(value);
+    if (!parsed) {
+      this.logger.warn(`[ContentResolver] Invalid reference format: ${value}`);
+      return value;
+    }
+
+    // Download content
+    try {
+      const storage = await this.getStorage();
+      const stream = await storage.client.getObject(parsed.bucket, parsed.path);
+      
+      const chunks = [];
+      for await (const chunk of stream) {
+        chunks.push(chunk);
+      }
+      
+      const content = Buffer.concat(chunks).toString('utf-8');
+      this.logger.debug?.(`[ContentResolver] Downloaded ${content.length} bytes from ${value}`);
+      
+      return content;
+    } catch (error) {
+      this.logger.error(`[ContentResolver] Failed to download ${value}: ${error.message}`);
+      throw new Error(`Failed to resolve content reference: ${error.message}`);
+    }
+  }
+
+  /**
+   * Store content if above threshold, otherwise return as-is
+   * @param {string} content - Text content to potentially store
+   * @param {Object} context - Workflow context for path generation
+   * @param {string} [context.workflow_id] - Workflow ID
+   * @param {string} [context.step_id] - Step ID
+   * @param {string} [filename] - Optional filename hint
+   * @returns {Promise<{ value: string, stored: boolean, size: number }>}
+   */
+  async store(content, context = {}, filename = null) {
+    if (!content || typeof content !== 'string') {
+      return { value: content, stored: false, size: 0 };
+    }
+
+    const size = Buffer.byteLength(content, 'utf-8');
+
+    // Below threshold - return as-is
+    if (size <= this.threshold) {
+      return { value: content, stored: false, size };
+    }
+
+    // Above threshold - store in MinIO
+    try {
+      const storage = await this.getStorage();
+      
+      const workflowId = context.workflow_id || 'standalone';
+      const stepId = context.step_id || 'content';
+      const pathPrefix = `content/${workflowId}/${stepId}`;
+      
+      const buffer = Buffer.from(content, 'utf-8');
+      const result = await storage.uploadWithFingerprint(
+        'workflow',
+        buffer,
+        pathPrefix,
+        filename ? filename.split('.').pop() : 'txt'
+      );
+      
+      const ref = `minio://workflow/${result.path}`;
+      this.logger.debug?.(`[ContentResolver] Stored ${size} bytes as ${ref}`);
+      
+      return { 
+        value: ref, 
+        stored: true, 
+        size,
+        fingerprint: result.fingerprint 
+      };
+    } catch (error) {
+      this.logger.error(`[ContentResolver] Failed to store content: ${error.message}`);
+      // Fallback: return original content if storage fails
+      this.logger.warn('[ContentResolver] Falling back to inline content');
+      return { value: content, stored: false, size, warning: error.message };
+    }
+  }
+
+  /**
+   * Process an input object - resolve all string fields that are references
+   * @param {Object} input - Input object with potential references
+   * @param {string[]} [fields] - Specific fields to resolve (default: all string fields)
+   * @returns {Promise<Object>} Input with resolved content
+   */
+  async resolveInput(input, fields = null) {
+    if (!input || typeof input !== 'object') {
+      return input;
+    }
+
+    const result = { ...input };
+    const fieldsToResolve = fields || Object.keys(input);
+
+    for (const field of fieldsToResolve) {
+      if (typeof result[field] === 'string' && isReference(result[field])) {
+        result[field] = await this.resolve(result[field]);
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Process an output object - store large string fields as references
+   * @param {Object} output - Output object with potential large content
+   * @param {Object} context - Workflow context
+   * @param {string[]} [fields] - Specific fields to process (default: all string fields)
+   * @returns {Promise<Object>} Output with large content stored as references
+   */
+  async storeOutput(output, context = {}, fields = null) {
+    if (!output || typeof output !== 'object') {
+      return output;
+    }
+
+    const result = { ...output };
+    const fieldsToProcess = fields || Object.keys(output);
+
+    for (const field of fieldsToProcess) {
+      if (typeof result[field] === 'string') {
+        const stored = await this.store(result[field], context, field);
+        result[field] = stored.value;
+        if (stored.stored) {
+          result[`${field}_stored`] = true;
+          result[`${field}_size`] = stored.size;
+        }
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Get current threshold
+   * @returns {number} Threshold in bytes
+   */
+  getThreshold() {
+    return this.threshold;
+  }
+
+  /**
+   * Set threshold
+   * @param {number} bytes - New threshold in bytes
+   */
+  setThreshold(bytes) {
+    this.threshold = bytes;
+  }
+}
+
+// Export class and utilities
+module.exports = ContentResolver;
+module.exports.ContentResolver = ContentResolver;
+module.exports.isReference = isReference;
+module.exports.parseReference = parseReference;
+module.exports.DEFAULT_THRESHOLD = DEFAULT_THRESHOLD;
+