@qelos/plugins-cli 0.0.21 → 0.0.23

package/commands/push.mjs

@@ -1,14 +1,14 @@
 import pushController from "../controllers/push.mjs";
 
-export default function createCommand(program) {
+export default function pushCommand(program) {
   program
-    .command('push [type] [path]', 'push to qelos app. Ability to push components, blueprints, configurations, plugins, blocks, and more.',
+    .command('push [type] [path]', 'push to qelos app. Ability to push components, blueprints, configurations, plugins, blocks, committed files, or staged files.',
       (yargs) => {
         return yargs
           .positional('type', {
-            describe: 'Type of the resource to push. Can be components, blueprints, configurations, plugins, blocks, or all.',
+            describe: 'Type of the resource to push. Can be components, blueprints, configurations, plugins, blocks, integrations, connections, committed, staged, or all.',
             type: 'string',
-            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'connections', 'all', '*'],
+            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'connections', 'committed', 'staged', 'all', '*'],
             required: true
           })
           .positional('path', {

package/controllers/push.mjs

@@ -6,12 +6,95 @@ import { pushPlugins } from '../services/plugins.mjs';
 import { pushBlocks } from '../services/blocks.mjs';
 import { pushIntegrations } from '../services/integrations.mjs';
 import { pushConnections } from '../services/connections.mjs';
+import { getGitFiles, prepareTempDirectories } from '../services/git-files.mjs';
 import { logger } from '../services/logger.mjs';
 import fs from 'node:fs';
 import path from 'node:path';
+import { mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
 
 export default async function pushController({ type, path: sourcePath }) {
+  let tempDir = null;
+  
   try {
+    // Handle git-based types (committed and staged)
+    if (type === 'committed' || type === 'staged') {
+      // Validate path exists and is a directory
+      if (!fs.existsSync(sourcePath)) {
+        logger.error(`Path does not exist: ${sourcePath}`);
+        logger.info('Please provide a valid directory path');
+        process.exit(1);
+      }
+
+      const stat = fs.statSync(sourcePath);
+      if (!stat.isDirectory()) {
+        logger.error(`For ${type} files, path must be a directory: ${sourcePath}`);
+        process.exit(1);
+      }
+
+      // Get and classify files from git
+      const classifiedFiles = getGitFiles(type, sourcePath);
+      
+      // Check if we have any files to push
+      const hasFiles = Object.values(classifiedFiles).some(files => files.length > 0);
+      if (!hasFiles) {
+        logger.info(`No ${type} files to push`);
+        return;
+      }
+
+      // Create temporary directory
+      tempDir = await mkdtemp(path.join(tmpdir(), 'qelos-push-'));
+      
+      // Prepare temporary directories and copy files
+      const tempPaths = prepareTempDirectories(classifiedFiles, tempDir);
+      
+      logger.section(`Pushing ${type} files from ${sourcePath}`);
+      
+      const sdk = await initializeSdk();
+      
+      // Push each type of file
+      const types = [
+        { name: 'components', fn: pushComponents },
+        { name: 'blueprints', fn: pushBlueprints },
+        { name: 'configs', fn: pushConfigurations },
+        { name: 'plugins', fn: pushPlugins },
+        { name: 'blocks', fn: pushBlocks },
+        { name: 'integrations', fn: pushIntegrations },
+        { name: 'connections', fn: pushConnections }
+      ];
+
+      for (const { name, fn } of types) {
+        if (!tempPaths[name]) continue;
+        
+        logger.section(`Pushing ${name} (${classifiedFiles[name].length} file(s))`);
+        
+        // Show the actual files being pushed
+        classifiedFiles[name].forEach(file => {
+          logger.step(`→ ${path.relative(sourcePath, file)}`);
+        });
+        
+        try {
+          await fn(sdk, tempPaths[name]);
+          logger.success(`Successfully pushed ${name}`);
+        } catch (error) {
+          logger.error(`Failed to push ${name}`, error);
+        }
+      }
+
+      // Handle special cases (prompts and micro-frontends)
+      if (classifiedFiles.prompts.length > 0) {
+        logger.info(`Found ${classifiedFiles.prompts.length} prompt file(s) that will be pushed via their parent integrations`);
+      }
+      
+      if (classifiedFiles.microFrontends.length > 0) {
+        logger.info(`Found ${classifiedFiles.microFrontends.length} micro-frontend HTML file(s) that will be pushed via their parent plugins`);
+      }
+
+      logger.success(`Successfully pushed ${type} files`);
+      return;
+    }
+
+    // Original logic for other types
     // Validate path exists
     if (!fs.existsSync(sourcePath)) {
       logger.error(`Path does not exist: ${sourcePath}`);
@@ -92,7 +175,7 @@ export default async function pushController({ type, path: sourcePath }) {
       await pushConfigurations(sdk, basePath, { targetFile });
     } else {
       logger.error(`Unknown type: ${type}`);
-      logger.info('Supported types: components, blueprints, plugins, blocks, integrations, connections, config, configs, configuration, all');
+      logger.info('Supported types: components, blueprints, plugins, blocks, integrations, connections, config, configs, configuration, committed, staged, all');
       process.exit(1);
     }
 
@@ -112,5 +195,15 @@ export default async function pushController({ type, path: sourcePath }) {
     }
     
     process.exit(1);
+  } finally {
+    // Clean up temporary directory if it was created
+    if (tempDir && fs.existsSync(tempDir)) {
+      try {
+        fs.rmSync(tempDir, { recursive: true, force: true });
+        logger.debug(`Cleaned up temporary directory: ${tempDir}`);
+      } catch (error) {
+        logger.warning(`Failed to clean up temporary directory: ${tempDir}`, error);
+      }
+    }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qelos/plugins-cli",
-  "version": "0.0.21",
+  "version": "0.0.23",
   "description": "CLI to manage QELOS plugins",
   "main": "cli.mjs",
   "bin": {

package/services/file-refs.mjs

@@ -134,26 +134,33 @@ export function extractIntegrationContent(integration, integrationPath, fileName
 
   // Extract the first pre_message content
   const preMessage = updatedIntegration.target.details.pre_messages[0];
-  if (preMessage && preMessage.content && typeof preMessage.content === 'string') {
-    const content = preMessage.content;
-    
-    // Skip if content is empty or just whitespace
-    if (!content.trim()) {
+  if (preMessage && preMessage.content) {
+    // Check if content is already a $ref
+    if (typeof preMessage.content === 'object' && preMessage.content.$ref) {
       return updatedIntegration;
     }
+    
+    // Only extract if content is a string
+    if (typeof preMessage.content === 'string') {
+      const content = preMessage.content;
+      
+      // Skip if content is empty or just whitespace
+      if (!content.trim()) {
+        return updatedIntegration;
+      }
 
-    // Generate filename
-    const baseName = path.basename(fileName, '.integration.json');
-    const mdFileName = `${baseName}.md`;
-    const mdFilePath = path.join(extractDir, mdFileName);
-    const relativeRef = `./prompts/${mdFileName}`;
+      // Generate filename
+      const baseName = path.basename(fileName, '.integration.json');
+      const mdFileName = `${baseName}.md`;
+      const mdFilePath = path.join(extractDir, mdFileName);
+      const relativeRef = `./prompts/${mdFileName}`;
 
-    // Write content to file
-    fs.writeFileSync(mdFilePath, content, 'utf-8');
-    logger.debug(`Extracted pre_message content to: ${relativeRef}`);
+      // Write content to file
+      fs.writeFileSync(mdFilePath, content, 'utf-8');
 
-    // Replace with $ref
-    updatedIntegration.target.details.pre_messages[0].content = { $ref: relativeRef };
+      // Replace with $ref
+      updatedIntegration.target.details.pre_messages[0].content = { $ref: relativeRef };
+    }
   }
 
   return updatedIntegration;

package/services/git-files.mjs

@@ -0,0 +1,288 @@
+import { execSync } from 'node:child_process';
+import { logger } from './logger.mjs';
+import path from 'node:path';
+import fs from 'node:fs';
+
+/**
+ * Get the list of files committed in the last commit
+ * @returns {string[]} Array of file paths
+ */
+function getCommittedFiles() {
+  try {
+    const output = execSync('git diff-tree --no-commit-id --name-only -r HEAD', { encoding: 'utf-8' });
+    return output.trim().split('\n').filter(file => file);
+  } catch (error) {
+    logger.error('Failed to get committed files', error);
+    throw new Error('Unable to retrieve committed files from git');
+  }
+}
+
+/**
+ * Get the list of staged files
+ * @returns {string[]} Array of file paths
+ */
+function getStagedFiles() {
+  try {
+    const output = execSync('git diff --cached --name-only', { encoding: 'utf-8' });
+    return output.trim().split('\n').filter(file => file);
+  } catch (error) {
+    logger.error('Failed to get staged files', error);
+    throw new Error('Unable to retrieve staged files from git');
+  }
+}
+
+/**
+ * Find integration files that reference a specific file via $ref
+ * @param {string} refPath - The referenced file path (relative)
+ * @param {string} basePath - Base path to search for integrations
+ * @returns {string[]} Array of integration file paths that reference the file
+ */
+function findReferencingIntegrations(refPath, basePath) {
+  const referencingIntegrations = [];
+  const integrationsDir = path.join(basePath, 'integrations');
+  
+  if (!fs.existsSync(integrationsDir)) {
+    return referencingIntegrations;
+  }
+  
+  const integrationFiles = fs.readdirSync(integrationsDir)
+    .filter(file => file.endsWith('.integration.json'));
+  
+  for (const file of integrationFiles) {
+    const filePath = path.join(integrationsDir, file);
+    try {
+      const content = fs.readFileSync(filePath, 'utf-8');
+      const integration = JSON.parse(content);
+      
+      // Check all $ref references in the integration
+      const refs = findAllRefs(integration);
+      
+      // Check if any ref matches our target path
+      // Normalize paths for comparison (handle ./ and different separators)
+      const normalizedRefPath = refPath.replace(/^\.\//, '').replace(/\\/g, '/');
+      
+      if (refs.some(ref => {
+        const normalizedRef = ref.replace(/^\.\//, '').replace(/\\/g, '/');
+        return normalizedRef === normalizedRefPath;
+      })) {
+        referencingIntegrations.push(filePath);
+      }
+    } catch (error) {
+      logger.debug(`Failed to parse integration ${file}: ${error.message}`);
+    }
+  }
+  
+  return referencingIntegrations;
+}
+
+/**
+ * Recursively find all $ref objects in an object
+ * @param {any} obj - Object to search
+ * @param {Array} refs - Array to collect found references (used internally)
+ * @returns {Array} Array of found $ref paths
+ */
+function findAllRefs(obj, refs = []) {
+  if (Array.isArray(obj)) {
+    obj.forEach(item => findAllRefs(item, refs));
+  } else if (obj && typeof obj === 'object') {
+    if (obj.$ref && typeof obj.$ref === 'string') {
+      refs.push(obj.$ref);
+    } else {
+      Object.values(obj).forEach(value => findAllRefs(value, refs));
+    }
+  }
+  return refs;
+}
+
+/**
+ * Classify files by their type based on their location and extension
+ * @param {string[]} files - Array of file paths
+ * @param {string} basePath - Base path to resolve relative paths from
+ * @returns {Object} Object with file paths grouped by type
+ */
+function classifyFiles(files, basePath) {
+  const classified = {
+    components: [],
+    blueprints: [],
+    configs: [],
+    plugins: [],
+    blocks: [],
+    integrations: [],
+    connections: [],
+    prompts: [], // For .md files in prompts directories
+    microFrontends: [] // For .html files
+  };
+
+  for (const file of files) {
+    // Make sure the file exists
+    const fullPath = path.resolve(basePath, file);
+    if (!fs.existsSync(fullPath)) {
+      logger.warning(`File not found, skipping: ${file}`);
+      continue;
+    }
+
+    const relativePath = path.relative(basePath, fullPath);
+    const dir = path.dirname(relativePath);
+    const ext = path.extname(fullPath);
+    const basename = path.basename(fullPath, ext);
+
+    // Check for specific file types
+    if (relativePath.includes('components/') && ext === '.vue') {
+      classified.components.push(fullPath);
+    } else if (relativePath.includes('blueprints/') && ext === '.json') {
+      classified.blueprints.push(fullPath);
+    } else if (relativePath.includes('configs/') && ext === '.json') {
+      classified.configs.push(fullPath);
+    } else if (relativePath.includes('plugins/') && ext === '.json') {
+      classified.plugins.push(fullPath);
+    } else if (relativePath.includes('blocks/') && ext === '.json') {
+      classified.blocks.push(fullPath);
+    } else if (relativePath.includes('integrations/') && ext === '.json') {
+      classified.integrations.push(fullPath);
+    } else if (relativePath.includes('connections/') && ext === '.json') {
+      classified.connections.push(fullPath);
+    } else if (dir.includes('prompts') && ext === '.md') {
+      // Find integrations that reference this prompt file
+      classified.prompts.push(fullPath);
+      
+      // The ref path should be relative to the integrations directory
+      // If file is integrations/prompts/file.md, ref should be ./prompts/file.md
+      const refPath = './' + path.relative('integrations', relativePath);
+      
+      const referencingIntegrations = findReferencingIntegrations(refPath, basePath);
+      
+      // Add the referencing integrations to the integrations list
+      for (const integrationPath of referencingIntegrations) {
+        if (!classified.integrations.includes(integrationPath)) {
+          classified.integrations.push(integrationPath);
+          logger.debug(`Found integration referencing ${relativePath}: ${path.basename(integrationPath)}`);
+        }
+      }
+    } else if (ext === '.html') {
+      // Find plugins that contain this HTML file (micro-frontends)
+      classified.microFrontends.push(fullPath);
+      
+      // For HTML files, we need to find which plugin contains them
+      // HTML files in plugins are typically part of the plugin structure
+      const pluginDir = path.dirname(fullPath);
+      const pluginJson = path.join(pluginDir, 'plugin.json');
+      
+      if (fs.existsSync(pluginJson)) {
+        // This HTML file is part of a plugin
+        if (!classified.plugins.includes(pluginJson)) {
+          classified.plugins.push(pluginJson);
+          logger.debug(`Found plugin containing HTML ${relativePath}: ${path.basename(pluginJson)}`);
+        }
+      }
+    } else {
+      logger.debug(`Unclassified file: ${relativePath}`);
+    }
+  }
+
+  return classified;
+}
+
+/**
+ * Get files from git (committed or staged) and classify them
+ * @param {string} type - 'committed' or 'staged'
+ * @param {string} basePath - Base path to resolve files from
+ * @returns {Object} Classified files object
+ */
+export function getGitFiles(type, basePath) {
+  if (type !== 'committed' && type !== 'staged') {
+    throw new Error('Type must be either "committed" or "staged"');
+  }
+
+  const files = type === 'committed' ? getCommittedFiles() : getStagedFiles();
+  
+  if (files.length === 0) {
+    logger.info(`No ${type} files found`);
+    return {};
+  }
+
+  logger.info(`Found ${files.length} ${type} file(s)`);
+  const classified = classifyFiles(files, basePath);
+
+  // Log what we found
+  Object.entries(classified).forEach(([key, value]) => {
+    if (value.length > 0) {
+      if (key === 'prompts' || key === 'microFrontends') {
+        logger.info(`  ${key}: ${value.length} file(s) (will be pushed via parent)`);
+      } else {
+        logger.info(`  ${key}: ${value.length} file(s)`);
+      }
+    }
+  });
+
+  return classified;
+}
+
+/**
+ * Create temporary directories for each type and copy files
+ * @param {Object} classifiedFiles - Object with classified file paths
+ * @param {string} tempDir - Temporary directory base path
+ * @returns {Object} Object with paths to temporary directories
+ */
+export function prepareTempDirectories(classifiedFiles, tempDir) {
+  const tempPaths = {};
+  const copiedRefs = new Set(); // Track which ref files have been copied
+
+  // Create temp directory structure
+  fs.mkdirSync(tempDir, { recursive: true });
+
+  for (const [type, files] of Object.entries(classifiedFiles)) {
+    if (files.length === 0) continue;
+
+    // Skip prompts and microFrontends as they are handled by their parents
+    if (type === 'prompts' || type === 'microFrontends') continue;
+
+    const typeDir = path.join(tempDir, type);
+    fs.mkdirSync(typeDir, { recursive: true });
+    tempPaths[type] = typeDir;
+
+    // Use a Set to avoid duplicate files
+    const uniqueFiles = [...new Set(files)];
+
+    // Copy files to temp directory
+    for (const file of uniqueFiles) {
+      const dest = path.join(typeDir, path.basename(file));
+      fs.copyFileSync(file, dest);
+      logger.debug(`Copied ${file} to ${dest}`);
+      
+      // If this is an integration, check for $ref files and copy them too
+      if (type === 'integrations' && file.endsWith('.integration.json')) {
+        try {
+          const content = fs.readFileSync(dest, 'utf-8');
+          const integration = JSON.parse(content);
+          const refs = findAllRefs(integration);
+          
+          for (const ref of refs) {
+            if (copiedRefs.has(ref)) continue;
+            
+            // Resolve the ref path relative to the original file location
+            const originalDir = path.dirname(file);
+            const refSourcePath = path.resolve(originalDir, ref);
+            
+            if (fs.existsSync(refSourcePath)) {
+              // Create the same directory structure in temp
+              // The ref is relative to the integration file, so we need to copy it to the same relative path
+              const refDestPath = path.join(tempDir, 'integrations', ref);
+              const refDestDir = path.dirname(refDestPath);
+              
+              fs.mkdirSync(refDestDir, { recursive: true });
+              fs.copyFileSync(refSourcePath, refDestPath);
+              copiedRefs.add(ref);
+              logger.debug(`Copied referenced file ${ref} from ${refSourcePath} to ${refDestPath}`);
+            } else {
+              logger.debug(`Referenced file not found: ${refSourcePath}`);
+            }
+          }
+        } catch (error) {
+          logger.debug(`Failed to process refs for ${path.basename(file)}: ${error.message}`);
+        }
+      }
+    }
+  }
+
+  return tempPaths;
+}

package/services/integrations.mjs

@@ -76,6 +76,24 @@ function sanitizeIntegrationForFile(integration) {
       delete sanitized[field];
     }
   });
+  
+  // Remove _id from internal objects
+  if (sanitized.trigger && sanitized.trigger._id) {
+    delete sanitized.trigger._id;
+  }
+  
+  if (sanitized.target && sanitized.target._id) {
+    delete sanitized.target._id;
+  }
+  
+  if (Array.isArray(sanitized.dataManipulation)) {
+    sanitized.dataManipulation.forEach(item => {
+      if (item._id) {
+        delete item._id;
+      }
+    });
+  }
+  
   return sanitized;
 }
 
@@ -177,8 +195,13 @@ export async function pushIntegrations(sdk, path, options = {}) {
         logger.success(`Created: ${displayName}`);
       }
 
+      // Re-extract content to files to maintain $ref structure
+      // This ensures pre_messages are stored as prompt md files after pushing
+      const processedResponse = extractIntegrationContent(response, path, file);
+      
       // Persist returned integration (with _id) back to disk
-      writeIntegrationFile(filePath, sanitizeIntegrationForFile(response));
+      writeIntegrationFile(filePath, sanitizeIntegrationForFile(processedResponse));
+      
       results.push({ status: 'fulfilled' });
     } catch (error) {
       logger.error(`Failed to push integration file ${file}`, error);

package/services/plugins.mjs

@@ -4,6 +4,48 @@ import { join } from 'node:path';
 import { logger } from './logger.mjs';
 import { extractMicroFrontendStructures, resolveMicroFrontendStructures } from './micro-frontends.mjs';
 
+function sanitizePluginForFile(plugin) {
+  const sanitized = JSON.parse(JSON.stringify(plugin));
+  
+  // Remove _id from internal objects in arrays
+  if (Array.isArray(sanitized.subscribedEvents)) {
+    sanitized.subscribedEvents.forEach(item => {
+      if (item._id) delete item._id;
+    });
+  }
+  
+  if (Array.isArray(sanitized.microFrontends)) {
+    sanitized.microFrontends.forEach(mfe => {
+      if (mfe._id) delete mfe._id;
+      if (Array.isArray(mfe.requires)) {
+        mfe.requires.forEach(item => {
+          if (item._id) delete item._id;
+        });
+      }
+    });
+  }
+  
+  if (Array.isArray(sanitized.injectables)) {
+    sanitized.injectables.forEach(item => {
+      if (item._id) delete item._id;
+    });
+  }
+  
+  if (Array.isArray(sanitized.navBarGroups)) {
+    sanitized.navBarGroups.forEach(item => {
+      if (item._id) delete item._id;
+    });
+  }
+  
+  if (Array.isArray(sanitized.cruds)) {
+    sanitized.cruds.forEach(item => {
+      if (item._id) delete item._id;
+    });
+  }
+  
+  return sanitized;
+}
+
 /**
  * Push plugins from local directory to remote
  * @param {Object} sdk - Initialized SDK instance
@@ -179,7 +221,10 @@ export async function pullPlugins(sdk, targetPath) {
       cruds: (fullPlugin.cruds || []).map(removeIdFromObject),
     }
     
-    fs.writeFileSync(filePath, JSON.stringify(relevantFields, null, 2), 'utf-8');
+    // Sanitize the plugin to remove any remaining _id fields from internal objects
+    const sanitizedPlugin = sanitizePluginForFile(relevantFields);
+    
+    fs.writeFileSync(filePath, JSON.stringify(sanitizedPlugin, null, 2), 'utf-8');
     logger.step(`Pulled: ${plugin.apiPath}`);
   }));