@qelos/plugins-cli 0.0.20 → 0.0.22

package/commands/pull.mjs

@@ -8,7 +8,7 @@ export default function pullCommand(program) {
           .positional('type', {
             describe: 'Type of the resource to pull. Can be components, blueprints, configurations, plugins, blocks, or all.',
             type: 'string',
-            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'all', '*'],
+            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'connections', 'all', '*'],
             required: true
           })
           .positional('path', {

package/commands/push.mjs

@@ -8,7 +8,7 @@ export default function createCommand(program) {
           .positional('type', {
             describe: 'Type of the resource to push. Can be components, blueprints, configurations, plugins, blocks, or all.',
             type: 'string',
-            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'all', '*'],
+            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'connections', 'all', '*'],
             required: true
           })
           .positional('path', {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qelos/plugins-cli",
-  "version": "0.0.20",
+  "version": "0.0.22",
   "description": "CLI to manage QELOS plugins",
   "main": "cli.mjs",
   "bin": {

package/services/components.mjs

@@ -50,22 +50,29 @@ export async function pushComponents(sdk, path, options = {}) {
         component => component.identifier === targetIdentifier || component.componentName === componentName
       );
       
-      if (existingComponent) {
-        await sdk.components.update(existingComponent._id, {
-          identifier: targetIdentifier,
-          componentName: componentName,
-          content,
-          description: info.description || existingComponent.description || 'Component description'
-        });
-        logger.success(`Updated: ${componentName}`);
-      } else {
-        await sdk.components.create({
-          identifier: targetIdentifier,
-          componentName: componentName,
-          content,
-          description: targetDescription
-        });
-        logger.success(`Created: ${componentName}`);
+      try {
+        if (existingComponent) {
+          await sdk.components.update(existingComponent._id, {
+            identifier: targetIdentifier,
+            componentName: componentName,
+            content,
+            description: info.description || existingComponent.description || 'Component description'
+          });
+          logger.success(`Updated: ${componentName}`);
+        } else {
+          await sdk.components.create({
+            identifier: targetIdentifier,
+            componentName: componentName,
+            content,
+            description: targetDescription
+          });
+          logger.success(`Created: ${componentName}`);
+        }
+      } catch (error) {
+        // Extract reason from error details if available
+        const reason = error.details?.reason || error.message;
+        logger.error(`Failed to push component: ${componentName} - ${reason}`);
+        throw error;
       }
     }
   }));

package/services/configurations.mjs

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import { join } from 'node:path';
 import { logger } from './logger.mjs';
 import { appUrl } from './sdk.mjs';
+import { extractConfigContent, resolveReferences } from './file-refs.mjs';
 
 /**
  * Push configurations from local directory to remote
@@ -45,6 +46,9 @@ export async function pushConfigurations(sdk, path, options = {}) {
       
       logger.step(`Pushing configuration: ${key}`);
       
+      // Resolve any $ref references in the configuration
+      configData = await resolveReferences(configData, path);
+      
       // Special handling for app-configuration: ensure QELOS_URL hostname is in websiteUrls
       if (key === 'app-configuration') {
         try {
@@ -160,7 +164,10 @@ export async function pullConfigurations(sdk, targetPath) {
     // Remove fields that shouldn't be in the file
     const { _id, tenant, created, updated, ...relevantFields } = fullConfig;
     
-    fs.writeFileSync(filePath, JSON.stringify(relevantFields, null, 2), 'utf-8');
+    // Extract content to files for supported config types
+    const processedConfig = extractConfigContent(relevantFields, targetPath);
+    
+    fs.writeFileSync(filePath, JSON.stringify(processedConfig, null, 2), 'utf-8');
     logger.step(`Pulled: ${config.key}`);
   }));
 

package/services/file-refs.mjs

@@ -0,0 +1,263 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { logger } from './logger.mjs';
+
+/**
+ * Configuration mapping for which fields should be extracted to files for each config type
+ */
+const CONFIG_EXTRACTION_MAP = {
+  'ssr-scripts': {
+    fields: ['head', 'body'],
+    fileExtension: '.html',
+    subdirectory: 'html'
+  },
+  'users-header': {
+    fields: ['html'],
+    fileExtension: '.html',
+    subdirectory: 'html'
+  }
+};
+
+/**
+ * Check if an integration is an AI agent that should have its pre_messages extracted
+ * @param {Object} integration - Integration object
+ * @returns {boolean} True if this is an AI agent with chatCompletion
+ */
+function isAiAgent(integration) {
+  return (
+    integration &&
+    Array.isArray(integration.kind) &&
+    integration.kind.includes('qelos') &&
+    integration.trigger?.operation === 'chatCompletion' &&
+    integration.target?.operation === 'chatCompletion' &&
+    integration.target?.details?.pre_messages?.length > 0
+  );
+}
+
+/**
+ * Convert a string to kebab-case for filenames
+ * @param {string} str - String to convert
+ * @returns {string} Kebab-cased string
+ */
+function toKebabCase(str) {
+  return str
+    .replace(/([a-z])([A-Z])/g, '$1-$2')
+    .replace(/[\s_]+/g, '-')
+    .toLowerCase();
+}
+
+/**
+ * Load content from a $ref reference
+ * @param {string} ref - Reference path (relative, absolute, or URL)
+ * @param {string} basePath - Base path for resolving relative references
+ * @returns {Promise<string>} Loaded content
+ */
+export async function loadReference(ref, basePath) {
+  // Handle HTTP/HTTPS URLs
+  if (ref.startsWith('http://') || ref.startsWith('https://')) {
+    logger.debug(`Loading reference from URL: ${ref}`);
+    try {
+      const response = await fetch(ref);
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      return await response.text();
+    } catch (error) {
+      throw new Error(`Failed to load from URL ${ref}: ${error.message}`);
+    }
+  }
+
+  // Handle absolute and relative paths
+  const filePath = path.isAbsolute(ref) 
+    ? ref 
+    : path.resolve(basePath, ref);
+
+  logger.debug(`Loading reference from file: ${filePath}`);
+  
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Referenced file does not exist: ${filePath}`);
+  }
+
+  return fs.readFileSync(filePath, 'utf-8');
+}
+
+/**
+ * Recursively resolve all $ref references in an object
+ * @param {any} obj - Object to resolve references in
+ * @param {string} basePath - Base path for resolving relative references
+ * @returns {Promise<any>} Object with all references resolved
+ */
+export async function resolveReferences(obj, basePath) {
+  if (Array.isArray(obj)) {
+    return await Promise.all(obj.map(item => resolveReferences(item, basePath)));
+  }
+  
+  if (obj && typeof obj === 'object') {
+    // Check if this is a $ref object
+    if (obj.$ref && typeof obj.$ref === 'string') {
+      return await loadReference(obj.$ref, basePath);
+    }
+    
+    // Otherwise, recursively resolve all properties
+    const resolved = {};
+    for (const [key, value] of Object.entries(obj)) {
+      resolved[key] = await resolveReferences(value, basePath);
+    }
+    return resolved;
+  }
+  
+  // Primitive value, return as-is
+  return obj;
+}
+
+/**
+ * Extract content from integration pre_messages to separate files
+ * @param {Object} integration - Integration object
+ * @param {string} integrationPath - Path where integration files are stored
+ * @param {string} fileName - Name of the integration file (without extension)
+ * @returns {Object} Updated integration with $ref objects
+ */
+export function extractIntegrationContent(integration, integrationPath, fileName) {
+  // Check if this is an AI agent
+  if (!isAiAgent(integration)) {
+    return integration;
+  }
+
+  const updatedIntegration = JSON.parse(JSON.stringify(integration)); // Deep clone
+  
+  // Create prompts subdirectory if needed
+  const extractDir = path.join(integrationPath, 'prompts');
+  if (!fs.existsSync(extractDir)) {
+    fs.mkdirSync(extractDir, { recursive: true });
+    logger.debug(`Created directory: ${extractDir}`);
+  }
+
+  // Extract the first pre_message content
+  const preMessage = updatedIntegration.target.details.pre_messages[0];
+  if (preMessage && preMessage.content) {
+    // Check if content is already a $ref
+    if (typeof preMessage.content === 'object' && preMessage.content.$ref) {
+      return updatedIntegration;
+    }
+    
+    // Only extract if content is a string
+    if (typeof preMessage.content === 'string') {
+      const content = preMessage.content;
+      
+      // Skip if content is empty or just whitespace
+      if (!content.trim()) {
+        return updatedIntegration;
+      }
+
+      // Generate filename
+      const baseName = path.basename(fileName, '.integration.json');
+      const mdFileName = `${baseName}.md`;
+      const mdFilePath = path.join(extractDir, mdFileName);
+      const relativeRef = `./prompts/${mdFileName}`;
+
+      // Write content to file
+      fs.writeFileSync(mdFilePath, content, 'utf-8');
+
+      // Replace with $ref
+      updatedIntegration.target.details.pre_messages[0].content = { $ref: relativeRef };
+    }
+  }
+
+  return updatedIntegration;
+}
+
+/**
+ * Extract all integrations that have content to be externalized
+ * @param {Array} integrations - Array of integration objects
+ * @param {string} integrationPath - Path where integration files are stored
+ * @returns {Array} Updated integrations with $ref objects
+ */
+export function extractAllIntegrationContent(integrations, integrationPath) {
+  return integrations.map((integration, index) => {
+    // Generate a filename for this integration
+    const displayName = integration?.trigger?.details?.name || 
+                       integration?.target?.details?.name || 
+                       integration?._id || 
+                       `integration-${index + 1}`;
+    const fileName = `${toKebabCase(displayName)}.integration.json`;
+    
+    return extractIntegrationContent(integration, integrationPath, fileName);
+  });
+}
+export function extractConfigContent(config, configPath) {
+  const extractionConfig = CONFIG_EXTRACTION_MAP[config.key];
+  
+  if (!extractionConfig || !config.metadata) {
+    return config;
+  }
+
+  const updatedConfig = JSON.parse(JSON.stringify(config)); // Deep clone
+  const { fields, fileExtension, subdirectory } = extractionConfig;
+  
+  // Create subdirectory if needed
+  const extractDir = path.join(configPath, subdirectory);
+  if (!fs.existsSync(extractDir)) {
+    fs.mkdirSync(extractDir, { recursive: true });
+    logger.debug(`Created directory: ${extractDir}`);
+  }
+
+  // Extract each field
+  for (const field of fields) {
+    if (updatedConfig.metadata[field] && typeof updatedConfig.metadata[field] === 'string') {
+      const content = updatedConfig.metadata[field];
+      
+      // Skip if content is empty or just whitespace
+      if (!content.trim()) {
+        continue;
+      }
+
+      // Generate filename - use config key if only one field, otherwise include field name
+      let fileName;
+      if (fields.length === 1) {
+        fileName = `${config.key}${fileExtension}`;
+      } else {
+        fileName = `${config.key}-${field}${fileExtension}`;
+      }
+      const filePath = path.join(extractDir, fileName);
+      const relativeRef = `./${subdirectory}/${fileName}`;
+
+      // Write content to file
+      fs.writeFileSync(filePath, content, 'utf-8');
+      logger.debug(`Extracted ${field} to: ${relativeRef}`);
+
+      // Replace with $ref
+      updatedConfig.metadata[field] = { $ref: relativeRef };
+    }
+  }
+
+  return updatedConfig;
+}
+
+/**
+ * Extract all configurations that have content to be externalized
+ * @param {Array} configs - Array of configuration objects
+ * @param {string} configPath - Path where config files are stored
+ * @returns {Array} Updated configurations with $ref objects
+ */
+export function extractAllConfigContent(configs, configPath) {
+  return configs.map(config => extractConfigContent(config, configPath));
+}
+
+/**
+ * Recursively find all $ref objects in an object
+ * @param {any} obj - Object to search
+ * @param {Array} refs - Array to collect found references (used internally)
+ * @returns {Array} Array of found $ref paths
+ */
+export function findAllRefs(obj, refs = []) {
+  if (Array.isArray(obj)) {
+    obj.forEach(item => findAllRefs(item, refs));
+  } else if (obj && typeof obj === 'object') {
+    if (obj.$ref && typeof obj.$ref === 'string') {
+      refs.push(obj.$ref);
+    } else {
+      Object.values(obj).forEach(value => findAllRefs(value, refs));
+    }
+  }
+  return refs;
+}

package/services/integrations.mjs

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import { join } from 'node:path';
 import { logger } from './logger.mjs';
+import { extractIntegrationContent, resolveReferences } from './file-refs.mjs';
 
 const INTEGRATION_FILE_EXTENSION = '.integration.json';
 const INTEGRATIONS_API_PATH = '/api/integrations';
@@ -75,6 +76,24 @@ function sanitizeIntegrationForFile(integration) {
       delete sanitized[field];
     }
   });
+  
+  // Remove _id from internal objects
+  if (sanitized.trigger && sanitized.trigger._id) {
+    delete sanitized.trigger._id;
+  }
+  
+  if (sanitized.target && sanitized.target._id) {
+    delete sanitized.target._id;
+  }
+  
+  if (Array.isArray(sanitized.dataManipulation)) {
+    sanitized.dataManipulation.forEach(item => {
+      if (item._id) {
+        delete item._id;
+      }
+    });
+  }
+  
   return sanitized;
 }
 
@@ -123,7 +142,11 @@ export async function pullIntegrations(sdk, targetPath) {
   integrations.forEach((integration, index) => {
     const fileName = buildFileName(integration, index, usedNames);
     const filePath = join(targetPath, fileName);
-    writeIntegrationFile(filePath, sanitizeIntegrationForFile(integration));
+    
+    // Extract content to files for AI agents
+    const processedIntegration = extractIntegrationContent(integration, targetPath, fileName);
+    
+    writeIntegrationFile(filePath, sanitizeIntegrationForFile(processedIntegration));
     logger.step(`Pulled: ${getIntegrationDisplayName(integration) || integration._id || fileName}`);
   });
 
@@ -154,8 +177,12 @@ export async function pushIntegrations(sdk, path, options = {}) {
     try {
       const integrationData = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
       validateIntegrationPayload(integrationData, file);
-      const payload = toRequestPayload(integrationData);
-      const displayName = getIntegrationDisplayName(integrationData) || file.replace(INTEGRATION_FILE_EXTENSION, '');
+      
+      // Resolve any $ref references in the integration
+      const resolvedIntegration = await resolveReferences(integrationData, path);
+      
+      const payload = toRequestPayload(resolvedIntegration);
+      const displayName = getIntegrationDisplayName(resolvedIntegration) || file.replace(INTEGRATION_FILE_EXTENSION, '');
 
       logger.step(`Pushing integration: ${displayName}`);
 
@@ -168,8 +195,13 @@ export async function pushIntegrations(sdk, path, options = {}) {
         logger.success(`Created: ${displayName}`);
       }
 
+      // Re-extract content to files to maintain $ref structure
+      // This ensures pre_messages are stored as prompt md files after pushing
+      const processedResponse = extractIntegrationContent(response, path, file);
+      
       // Persist returned integration (with _id) back to disk
-      writeIntegrationFile(filePath, sanitizeIntegrationForFile(response));
+      writeIntegrationFile(filePath, sanitizeIntegrationForFile(processedResponse));
+      
       results.push({ status: 'fulfilled' });
     } catch (error) {
       logger.error(`Failed to push integration file ${file}`, error);

package/services/micro-frontends.mjs

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { logger } from './logger.mjs';
+import { loadReference } from './file-refs.mjs';
 
 /**
  * Convert a string to kebab-case
@@ -59,41 +60,6 @@ export function extractMicroFrontendStructures(microFrontends, pluginPath) {
   });
 }
 
-/**
- * Load content from a $ref reference
- * @param {string} ref - Reference path (relative, absolute, or URL)
- * @param {string} basePath - Base path for resolving relative references
- * @returns {Promise<string>} Loaded content
- */
-async function loadReference(ref, basePath) {
-  // Handle HTTP/HTTPS URLs
-  if (ref.startsWith('http://') || ref.startsWith('https://')) {
-    logger.debug(`Loading structure from URL: ${ref}`);
-    try {
-      const response = await fetch(ref);
-      if (!response.ok) {
-        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-      }
-      return await response.text();
-    } catch (error) {
-      throw new Error(`Failed to load from URL ${ref}: ${error.message}`);
-    }
-  }
-
-  // Handle absolute and relative paths
-  const filePath = path.isAbsolute(ref) 
-    ? ref 
-    : path.resolve(basePath, ref);
-
-  logger.debug(`Loading structure from file: ${filePath}`);
-  
-  if (!fs.existsSync(filePath)) {
-    throw new Error(`Referenced file does not exist: ${filePath}`);
-  }
-
-  return fs.readFileSync(filePath, 'utf-8');
-}
-
 /**
  * Resolve micro-frontend structures from $ref references
  * @param {Array} microFrontends - Array of micro-frontend objects

package/services/plugins.mjs

@@ -4,6 +4,48 @@ import { join } from 'node:path';
 import { logger } from './logger.mjs';
 import { extractMicroFrontendStructures, resolveMicroFrontendStructures } from './micro-frontends.mjs';
 
+function sanitizePluginForFile(plugin) {
+  const sanitized = JSON.parse(JSON.stringify(plugin));
+  
+  // Remove _id from internal objects in arrays
+  if (Array.isArray(sanitized.subscribedEvents)) {
+    sanitized.subscribedEvents.forEach(item => {
+      if (item._id) delete item._id;
+    });
+  }
+  
+  if (Array.isArray(sanitized.microFrontends)) {
+    sanitized.microFrontends.forEach(mfe => {
+      if (mfe._id) delete mfe._id;
+      if (Array.isArray(mfe.requires)) {
+        mfe.requires.forEach(item => {
+          if (item._id) delete item._id;
+        });
+      }
+    });
+  }
+  
+  if (Array.isArray(sanitized.injectables)) {
+    sanitized.injectables.forEach(item => {
+      if (item._id) delete item._id;
+    });
+  }
+  
+  if (Array.isArray(sanitized.navBarGroups)) {
+    sanitized.navBarGroups.forEach(item => {
+      if (item._id) delete item._id;
+    });
+  }
+  
+  if (Array.isArray(sanitized.cruds)) {
+    sanitized.cruds.forEach(item => {
+      if (item._id) delete item._id;
+    });
+  }
+  
+  return sanitized;
+}
+
 /**
  * Push plugins from local directory to remote
  * @param {Object} sdk - Initialized SDK instance
@@ -179,7 +221,10 @@ export async function pullPlugins(sdk, targetPath) {
       cruds: (fullPlugin.cruds || []).map(removeIdFromObject),
     }
     
-    fs.writeFileSync(filePath, JSON.stringify(relevantFields, null, 2), 'utf-8');
+    // Sanitize the plugin to remove any remaining _id fields from internal objects
+    const sanitizedPlugin = sanitizePluginForFile(relevantFields);
+    
+    fs.writeFileSync(filePath, JSON.stringify(sanitizedPlugin, null, 2), 'utf-8');
     logger.step(`Pulled: ${plugin.apiPath}`);
   }));