@qelos/plugins-cli 0.0.19 → 0.0.21

package/README.md

@@ -47,7 +47,7 @@ qplay create my-app
 
 ### Pull
 
-Pull resources from your Qelos instance to your local filesystem. This allows you to work on components, plugins, integrations, and blueprints locally.
+Pull resources from your Qelos instance to your local filesystem. This allows you to work on components, blueprints, configs, plugins, blocks, integrations, and connections locally.
 
 **Syntax:**
 ```bash
@@ -55,7 +55,7 @@ qelos pull <type> <path>
 ```
 
 **Arguments:**
-- `type` - Type of resource to pull (e.g., `components`, `plugins`, `integrations`, `blueprints`)
+- `type` - Type of resource to pull (e.g., `components`, `plugins`, `integrations`, `connections`, `blueprints`)
 - `path` - Local directory path where resources will be saved
 
 **Example - Pull Components:**
@@ -82,7 +82,7 @@ All 5 components pulled to ./my-components
 
 ### Push
 
-Push local resources to your Qelos instance. This allows you to update or create components, plugins, integrations, and blueprints from your local filesystem.
+Push local resources to your Qelos instance. This allows you to update or create components, blueprints, configs, plugins, blocks, integrations, and connections from your local filesystem.
 
 **Syntax:**
 ```bash
@@ -90,7 +90,7 @@ qelos push <type> <path>
 ```
 
 **Arguments:**
-- `type` - Type of resource to push (e.g., `components`, `plugins`, `integrations`, `blueprints`)
+- `type` - Type of resource to push (e.g., `components`, `plugins`, `integrations`, `connections`, `blueprints`)
 - `path` - Local directory path containing the resources to push
 
 **Example - Push Components:**
@@ -129,6 +129,39 @@ qelos pull components ./local-components
 qelos push components ./local-components
 ```
 
+### Integrations
+
+- Pulled integrations are stored as `.integration.json` files that exclude server-only fields such as `tenant`, `user`, `created`, `updated`, and `__v`.  
+- When pushing, the CLI automatically recalculates backend-only properties (like `kind`) so you do not need to keep them in local files.
+
+### Connections (Integration Sources)
+
+Connections (integration sources) are now fully supported via `qelos pull connections <path>` and `qelos push connections <path>`.
+
+- Each connection is stored as `<name>.connection.json` containing `name`, `kind`, `labels`, `metadata`, and an `authentication` placeholder:
+
+```json
+{
+  "_id": "64f1...",
+  "name": "OpenAI",
+  "kind": "openai",
+  "labels": ["ai"],
+  "metadata": { "defaultModel": "gpt-4o" },
+  "authentication": {
+    "$var": "INTEGRATION_AUTH_OPENAI"
+  }
+}
+```
+
+- At push time, the CLI reads the referenced environment variable (e.g., `INTEGRATION_AUTH_OPENAI`) which must contain a JSON string with the real credentials:
+
+```bash
+export INTEGRATION_AUTH_OPENAI='{"token":"sk-..."}'
+```
+
+- If the env var is missing, the CLI skips updating the secure authentication payload and only syncs metadata.
+- When new connections are created, the CLI persists the returned `_id` so future pushes update the same record.
+
 ## Help
 
 View all available commands and options:

package/commands/pull.mjs

@@ -8,7 +8,7 @@ export default function pullCommand(program) {
           .positional('type', {
             describe: 'Type of the resource to pull. Can be components, blueprints, configurations, plugins, blocks, or all.',
             type: 'string',
-            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'all', '*'],
+            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'connections', 'all', '*'],
             required: true
           })
           .positional('path', {

package/commands/push.mjs

@@ -8,7 +8,7 @@ export default function createCommand(program) {
           .positional('type', {
             describe: 'Type of the resource to push. Can be components, blueprints, configurations, plugins, blocks, or all.',
             type: 'string',
-            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'all', '*'],
+            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'connections', 'all', '*'],
             required: true
           })
           .positional('path', {

package/controllers/pull.mjs

@@ -4,6 +4,8 @@ import { pullBlueprints } from '../services/blueprints.mjs';
 import { pullConfigurations } from '../services/configurations.mjs';
 import { pullPlugins } from '../services/plugins.mjs';
 import { pullBlocks } from '../services/blocks.mjs';
+import { pullIntegrations } from '../services/integrations.mjs';
+import { pullConnections } from '../services/connections.mjs';
 import { logger } from '../services/logger.mjs';
 import fs from 'node:fs';
 import path from 'node:path';
@@ -36,7 +38,9 @@ export default async function pullController({ type, path: targetPath = './' })
         { name: 'blueprints', fn: pullBlueprints },
         { name: 'configs', fn: pullConfigurations },
         { name: 'plugins', fn: pullPlugins },
-        { name: 'blocks', fn: pullBlocks }
+        { name: 'blocks', fn: pullBlocks },
+        { name: 'integrations', fn: pullIntegrations },
+        { name: 'connections', fn: pullConnections }
       ];
 
       for (const { name, fn } of types) {
@@ -64,11 +68,15 @@ export default async function pullController({ type, path: targetPath = './' })
       await pullPlugins(sdk, targetPath);
     } else if (type === 'blocks') {
       await pullBlocks(sdk, targetPath);
+    } else if (type === 'integrations' || type === 'integration') {
+      await pullIntegrations(sdk, targetPath);
+    } else if (type === 'connections' || type === 'connection') {
+      await pullConnections(sdk, targetPath);
     } else if (type === 'config' || type === 'configs' || type === 'configuration') {
       await pullConfigurations(sdk, targetPath);
     } else {
       logger.error(`Unknown type: ${type}`);
-      logger.info('Supported types: components, blueprints, plugins, blocks, config, configs, configuration, all');
+      logger.info('Supported types: components, blueprints, plugins, blocks, integrations, connections, config, configs, configuration, all');
       process.exit(1);
     }
 

package/controllers/push.mjs

@@ -4,6 +4,8 @@ import { pushBlueprints } from '../services/blueprints.mjs';
 import { pushConfigurations } from '../services/configurations.mjs';
 import { pushPlugins } from '../services/plugins.mjs';
 import { pushBlocks } from '../services/blocks.mjs';
+import { pushIntegrations } from '../services/integrations.mjs';
+import { pushConnections } from '../services/connections.mjs';
 import { logger } from '../services/logger.mjs';
 import fs from 'node:fs';
 import path from 'node:path';
@@ -45,7 +47,9 @@ export default async function pushController({ type, path: sourcePath }) {
         { name: 'blueprints', fn: pushBlueprints },
         { name: 'configs', fn: pushConfigurations },
         { name: 'plugins', fn: pushPlugins },
-        { name: 'blocks', fn: pushBlocks }
+        { name: 'blocks', fn: pushBlocks },
+        { name: 'integrations', fn: pushIntegrations },
+        { name: 'connections', fn: pushConnections }
       ];
 
       for (const { name, fn } of types) {
@@ -80,11 +84,15 @@ export default async function pushController({ type, path: sourcePath }) {
       await pushPlugins(sdk, basePath, { targetFile });
     } else if (type === 'blocks') {
       await pushBlocks(sdk, basePath, { targetFile });
-    }  else if (type === 'config' || type === 'configs' || type === 'configuration') {
+    } else if (type === 'integrations' || type === 'integration') {
+      await pushIntegrations(sdk, basePath, { targetFile });
+    } else if (type === 'connections' || type === 'connection') {
+      await pushConnections(sdk, basePath, { targetFile });
+    } else if (type === 'config' || type === 'configs' || type === 'configuration') {
       await pushConfigurations(sdk, basePath, { targetFile });
     } else {
       logger.error(`Unknown type: ${type}`);
-      logger.info('Supported types: components, blueprints, plugins, blocks, config, configs, configuration, all');
+      logger.info('Supported types: components, blueprints, plugins, blocks, integrations, connections, config, configs, configuration, all');
       process.exit(1);
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qelos/plugins-cli",
-  "version": "0.0.19",
+  "version": "0.0.21",
   "description": "CLI to manage QELOS plugins",
   "main": "cli.mjs",
   "bin": {

package/services/blueprint-sdk-guides.mjs

@@ -68,12 +68,12 @@ function buildMarkdown({ blueprint, interfaceName, entityVarName, interfaceDefin
     '',
     '### Create an Entity',
     '```ts',
-    `const created = await ${entityVarName}.create(${exampleLiteral});`,
+    `const created = await ${entityVarName}.create({\n  metadata: ${indentLiteral(exampleLiteral, 2)},\n});`,
     '```',
     '',
     '### Update an Entity',
     '```ts',
-    `const updated = await ${entityVarName}.update('replace-with-entity-id', {\n  ...${exampleLiteral.replace(/\n/g, '\n  ')},\n});`,
+    `const updated = await ${entityVarName}.update('replace-with-entity-id', {\n  metadata: {\n    ...oldMetadata,\n    ...${indentLiteral(exampleLiteral, 4)},\n  },\n});`,
     '```',
     '',
     '### Delete an Entity',
@@ -219,6 +219,11 @@ function stringifyObjectLiteral(value, level = 0) {
   return String(value);
 }
 
+function indentLiteral(literal, spaces) {
+  const indent = ' '.repeat(spaces);
+  return literal.replace(/\n/g, `\n${indent}`);
+}
+
 function toCamelCase(value) {
   return (value || '')
     .replace(/[-_\s]+(.)?/g, (_, chr) => (chr ? chr.toUpperCase() : ''))

package/services/components.mjs

@@ -50,22 +50,29 @@ export async function pushComponents(sdk, path, options = {}) {
         component => component.identifier === targetIdentifier || component.componentName === componentName
       );
       
-      if (existingComponent) {
-        await sdk.components.update(existingComponent._id, {
-          identifier: targetIdentifier,
-          componentName: componentName,
-          content,
-          description: info.description || existingComponent.description || 'Component description'
-        });
-        logger.success(`Updated: ${componentName}`);
-      } else {
-        await sdk.components.create({
-          identifier: targetIdentifier,
-          componentName: componentName,
-          content,
-          description: targetDescription
-        });
-        logger.success(`Created: ${componentName}`);
+      try {
+        if (existingComponent) {
+          await sdk.components.update(existingComponent._id, {
+            identifier: targetIdentifier,
+            componentName: componentName,
+            content,
+            description: info.description || existingComponent.description || 'Component description'
+          });
+          logger.success(`Updated: ${componentName}`);
+        } else {
+          await sdk.components.create({
+            identifier: targetIdentifier,
+            componentName: componentName,
+            content,
+            description: targetDescription
+          });
+          logger.success(`Created: ${componentName}`);
+        }
+      } catch (error) {
+        // Extract reason from error details if available
+        const reason = error.details?.reason || error.message;
+        logger.error(`Failed to push component: ${componentName} - ${reason}`);
+        throw error;
       }
     }
   }));

package/services/configurations.mjs

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import { join } from 'node:path';
 import { logger } from './logger.mjs';
 import { appUrl } from './sdk.mjs';
+import { extractConfigContent, resolveReferences } from './file-refs.mjs';
 
 /**
  * Push configurations from local directory to remote
@@ -45,6 +46,9 @@ export async function pushConfigurations(sdk, path, options = {}) {
       
       logger.step(`Pushing configuration: ${key}`);
       
+      // Resolve any $ref references in the configuration
+      configData = await resolveReferences(configData, path);
+      
       // Special handling for app-configuration: ensure QELOS_URL hostname is in websiteUrls
       if (key === 'app-configuration') {
         try {
@@ -160,7 +164,10 @@ export async function pullConfigurations(sdk, targetPath) {
     // Remove fields that shouldn't be in the file
     const { _id, tenant, created, updated, ...relevantFields } = fullConfig;
     
-    fs.writeFileSync(filePath, JSON.stringify(relevantFields, null, 2), 'utf-8');
+    // Extract content to files for supported config types
+    const processedConfig = extractConfigContent(relevantFields, targetPath);
+    
+    fs.writeFileSync(filePath, JSON.stringify(processedConfig, null, 2), 'utf-8');
     logger.step(`Pulled: ${config.key}`);
   }));
 

package/services/connections.mjs

@@ -0,0 +1,239 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { logger } from './logger.mjs';
+
+const CONNECTION_FILE_EXTENSION = '.connection.json';
+const AUTH_PLACEHOLDER_KEY = '$var';
+
+function slugify(value = '') {
+  return value
+    .toString()
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+function ensureDirectory(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+    logger.info(`Created directory: ${dirPath}`);
+  }
+}
+
+function getDefaultAuthEnvVar(connection) {
+  if (connection?.authentication?.[AUTH_PLACEHOLDER_KEY]) {
+    return connection.authentication[AUTH_PLACEHOLDER_KEY];
+  }
+
+  if (connection?._id) {
+    return `INTEGRATION_AUTH_${connection._id}`;
+  }
+
+  const slug = slugify(connection?.name || 'connection').replace(/-/g, '_').toUpperCase();
+  const suffix = slug ? slug.slice(-6) : Math.random().toString(36).slice(-6).toUpperCase();
+  return `INTEGRATION_AUTH_${suffix || 'NEW'}`;
+}
+
+function buildFileName(connection, index, usedNames) {
+  const base =
+    slugify(connection?.name) ||
+    (connection?._id ? slugify(connection._id) : '') ||
+    `connection-${index + 1}`;
+
+  let fileName = base;
+  while (usedNames.has(fileName)) {
+    fileName = `${base}-${usedNames.size + 1}`;
+  }
+  usedNames.add(fileName);
+  return `${fileName}${CONNECTION_FILE_EXTENSION}`;
+}
+
+function sanitizeConnectionForFile(connection) {
+  return {
+    _id: connection._id,
+    name: connection.name,
+    kind: connection.kind,
+    labels: connection.labels || [],
+    metadata: connection.metadata || {},
+    authentication: {
+      [AUTH_PLACEHOLDER_KEY]: getDefaultAuthEnvVar(connection),
+    },
+  };
+}
+
+function writeConnectionFile(filePath, connection) {
+  fs.writeFileSync(filePath, JSON.stringify(connection, null, 2), 'utf-8');
+}
+
+function validateConnectionPayload(data, fileName) {
+  if (!data?.name) {
+    throw new Error(`Connection file ${fileName} must include a name`);
+  }
+
+  if (!data?.kind) {
+    throw new Error(`Connection file ${fileName} must include a kind`);
+  }
+
+  if (!data?.metadata || typeof data.metadata !== 'object') {
+    throw new Error(`Connection file ${fileName} must include metadata object`);
+  }
+
+  if (data.labels && !Array.isArray(data.labels)) {
+    throw new Error(`Connection file ${fileName} has invalid labels (must be an array)`);
+  }
+}
+
+function extractAuthenticationPayload(connectionFile, fileName) {
+  const authField = connectionFile.authentication;
+
+  if (!authField || typeof authField !== 'object') {
+    return { payload: undefined, envVar: null };
+  }
+
+  const envVarName = authField[AUTH_PLACEHOLDER_KEY];
+
+  if (!envVarName) {
+    throw new Error(
+      `Connection file ${fileName} must define authentication as { "${AUTH_PLACEHOLDER_KEY}": "INTEGRATION_AUTH_..." }`
+    );
+  }
+
+  const raw = process.env[envVarName];
+
+  if (!raw) {
+    if (!connectionFile._id) {
+      throw new Error(
+        `Environment variable ${envVarName} is required to create connection defined in ${fileName}`
+      );
+    }
+
+    logger.info(
+      `Skipping authentication update for ${connectionFile.name} (env ${envVarName} not set)`
+    );
+    return { payload: undefined, envVar: envVarName };
+  }
+
+  try {
+    const parsed = JSON.parse(raw);
+    if (typeof parsed !== 'object' || !parsed) {
+      throw new Error('Authentication env must contain a JSON object');
+    }
+    return { payload: parsed, envVar: envVarName };
+  } catch (error) {
+    throw new Error(
+      `Failed to parse JSON from env ${envVarName} for connection ${fileName}: ${error.message}`
+    );
+  }
+}
+
+export async function pullConnections(sdk, targetPath) {
+  ensureDirectory(targetPath);
+
+  let connections = [];
+  try {
+    connections = await sdk.integrationSources.getList();
+  } catch (error) {
+    logger.error('Failed to fetch integration sources', error);
+    throw error;
+  }
+
+  if (!Array.isArray(connections) || connections.length === 0) {
+    logger.warning('No connections found to pull');
+    return;
+  }
+
+  logger.info(`Found ${connections.length} connection(s) to pull`);
+  const usedNames = new Set();
+
+  connections.forEach((connection, index) => {
+    const fileName = buildFileName(connection, index, usedNames);
+    const filePath = path.join(targetPath, fileName);
+    writeConnectionFile(filePath, sanitizeConnectionForFile(connection));
+    logger.step(`Pulled connection: ${connection.name || connection._id}`);
+  });
+
+  logger.info(`Pulled ${connections.length} connection(s)`);
+}
+
+export async function pushConnections(sdk, sourcePath, options = {}) {
+  const { targetFile } = options;
+  const directoryFiles = fs.readdirSync(sourcePath);
+  const files = targetFile ? [targetFile] : directoryFiles;
+  const connectionFiles = files.filter((file) => file.endsWith(CONNECTION_FILE_EXTENSION));
+
+  if (connectionFiles.length === 0) {
+    if (targetFile) {
+      logger.warning(
+        `File ${targetFile} is not a ${CONNECTION_FILE_EXTENSION} connection file. Skipping.`
+      );
+    } else {
+      logger.warning(`No connection files (*${CONNECTION_FILE_EXTENSION}) found in ${sourcePath}`);
+    }
+    return;
+  }
+
+  logger.info(`Found ${connectionFiles.length} connection(s) to push`);
+
+  const results = [];
+
+  for (const file of connectionFiles) {
+    const filePath = path.join(sourcePath, file);
+
+    try {
+      const connectionData = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+      validateConnectionPayload(connectionData, file);
+
+      const { payload: authentication, envVar } = extractAuthenticationPayload(connectionData, file);
+
+      const requestBody = {
+        name: connectionData.name,
+        kind: connectionData.kind,
+        labels: connectionData.labels || [],
+        metadata: connectionData.metadata || {},
+        ...(authentication ? { authentication } : {}),
+      };
+
+      logger.step(`Pushing connection: ${connectionData.name}`);
+      let response;
+
+      if (connectionData._id) {
+        response = await sdk.integrationSources.update(connectionData._id, requestBody);
+        logger.success(`Updated connection: ${connectionData.name}`);
+      } else {
+        response = await sdk.integrationSources.create(requestBody);
+        logger.success(`Created connection: ${connectionData.name}`);
+      }
+
+      const authPlaceholder = connectionData.authentication?.[AUTH_PLACEHOLDER_KEY]
+        ? connectionData.authentication
+        : { [AUTH_PLACEHOLDER_KEY]: envVar || getDefaultAuthEnvVar(response) };
+
+      const fileContent = {
+        _id: response._id,
+        name: response.name,
+        kind: response.kind,
+        labels: response.labels || [],
+        metadata: response.metadata || {},
+        authentication: authPlaceholder,
+      };
+
+      writeConnectionFile(filePath, fileContent);
+      results.push({ status: 'fulfilled' });
+    } catch (error) {
+      logger.error(`Failed to push connection file ${file}`, error);
+      results.push({ status: 'rejected', reason: error });
+    }
+  }
+
+  const failures = results.filter((result) => result.status === 'rejected');
+  if (failures.length) {
+    logger.error(`\n${failures.length} connection(s) failed to push:`);
+    failures.forEach((failure) => {
+      logger.error(`  • ${failure.reason?.message || 'Unknown error'}`);
+    });
+    throw new Error(`Failed to push ${failures.length} connection(s)`);
+  }
+
+  logger.info(`Pushed ${connectionFiles.length} connection(s)`);
+}

package/services/file-refs.mjs

@@ -0,0 +1,256 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { logger } from './logger.mjs';
+
+/**
+ * Configuration mapping for which fields should be extracted to files for each config type
+ */
+const CONFIG_EXTRACTION_MAP = {
+  'ssr-scripts': {
+    fields: ['head', 'body'],
+    fileExtension: '.html',
+    subdirectory: 'html'
+  },
+  'users-header': {
+    fields: ['html'],
+    fileExtension: '.html',
+    subdirectory: 'html'
+  }
+};
+
+/**
+ * Check if an integration is an AI agent that should have its pre_messages extracted
+ * @param {Object} integration - Integration object
+ * @returns {boolean} True if this is an AI agent with chatCompletion
+ */
+function isAiAgent(integration) {
+  return (
+    integration &&
+    Array.isArray(integration.kind) &&
+    integration.kind.includes('qelos') &&
+    integration.trigger?.operation === 'chatCompletion' &&
+    integration.target?.operation === 'chatCompletion' &&
+    integration.target?.details?.pre_messages?.length > 0
+  );
+}
+
+/**
+ * Convert a string to kebab-case for filenames
+ * @param {string} str - String to convert
+ * @returns {string} Kebab-cased string
+ */
+function toKebabCase(str) {
+  return str
+    .replace(/([a-z])([A-Z])/g, '$1-$2')
+    .replace(/[\s_]+/g, '-')
+    .toLowerCase();
+}
+
+/**
+ * Load content from a $ref reference
+ * @param {string} ref - Reference path (relative, absolute, or URL)
+ * @param {string} basePath - Base path for resolving relative references
+ * @returns {Promise<string>} Loaded content
+ */
+export async function loadReference(ref, basePath) {
+  // Handle HTTP/HTTPS URLs
+  if (ref.startsWith('http://') || ref.startsWith('https://')) {
+    logger.debug(`Loading reference from URL: ${ref}`);
+    try {
+      const response = await fetch(ref);
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      return await response.text();
+    } catch (error) {
+      throw new Error(`Failed to load from URL ${ref}: ${error.message}`);
+    }
+  }
+
+  // Handle absolute and relative paths
+  const filePath = path.isAbsolute(ref) 
+    ? ref 
+    : path.resolve(basePath, ref);
+
+  logger.debug(`Loading reference from file: ${filePath}`);
+  
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Referenced file does not exist: ${filePath}`);
+  }
+
+  return fs.readFileSync(filePath, 'utf-8');
+}
+
+/**
+ * Recursively resolve all $ref references in an object
+ * @param {any} obj - Object to resolve references in
+ * @param {string} basePath - Base path for resolving relative references
+ * @returns {Promise<any>} Object with all references resolved
+ */
+export async function resolveReferences(obj, basePath) {
+  if (Array.isArray(obj)) {
+    return await Promise.all(obj.map(item => resolveReferences(item, basePath)));
+  }
+  
+  if (obj && typeof obj === 'object') {
+    // Check if this is a $ref object
+    if (obj.$ref && typeof obj.$ref === 'string') {
+      return await loadReference(obj.$ref, basePath);
+    }
+    
+    // Otherwise, recursively resolve all properties
+    const resolved = {};
+    for (const [key, value] of Object.entries(obj)) {
+      resolved[key] = await resolveReferences(value, basePath);
+    }
+    return resolved;
+  }
+  
+  // Primitive value, return as-is
+  return obj;
+}
+
+/**
+ * Extract content from integration pre_messages to separate files
+ * @param {Object} integration - Integration object
+ * @param {string} integrationPath - Path where integration files are stored
+ * @param {string} fileName - Name of the integration file (without extension)
+ * @returns {Object} Updated integration with $ref objects
+ */
+export function extractIntegrationContent(integration, integrationPath, fileName) {
+  // Check if this is an AI agent
+  if (!isAiAgent(integration)) {
+    return integration;
+  }
+
+  const updatedIntegration = JSON.parse(JSON.stringify(integration)); // Deep clone
+  
+  // Create prompts subdirectory if needed
+  const extractDir = path.join(integrationPath, 'prompts');
+  if (!fs.existsSync(extractDir)) {
+    fs.mkdirSync(extractDir, { recursive: true });
+    logger.debug(`Created directory: ${extractDir}`);
+  }
+
+  // Extract the first pre_message content
+  const preMessage = updatedIntegration.target.details.pre_messages[0];
+  if (preMessage && preMessage.content && typeof preMessage.content === 'string') {
+    const content = preMessage.content;
+    
+    // Skip if content is empty or just whitespace
+    if (!content.trim()) {
+      return updatedIntegration;
+    }
+
+    // Generate filename
+    const baseName = path.basename(fileName, '.integration.json');
+    const mdFileName = `${baseName}.md`;
+    const mdFilePath = path.join(extractDir, mdFileName);
+    const relativeRef = `./prompts/${mdFileName}`;
+
+    // Write content to file
+    fs.writeFileSync(mdFilePath, content, 'utf-8');
+    logger.debug(`Extracted pre_message content to: ${relativeRef}`);
+
+    // Replace with $ref
+    updatedIntegration.target.details.pre_messages[0].content = { $ref: relativeRef };
+  }
+
+  return updatedIntegration;
+}
+
+/**
+ * Extract all integrations that have content to be externalized
+ * @param {Array} integrations - Array of integration objects
+ * @param {string} integrationPath - Path where integration files are stored
+ * @returns {Array} Updated integrations with $ref objects
+ */
+export function extractAllIntegrationContent(integrations, integrationPath) {
+  return integrations.map((integration, index) => {
+    // Generate a filename for this integration
+    const displayName = integration?.trigger?.details?.name || 
+                       integration?.target?.details?.name || 
+                       integration?._id || 
+                       `integration-${index + 1}`;
+    const fileName = `${toKebabCase(displayName)}.integration.json`;
+    
+    return extractIntegrationContent(integration, integrationPath, fileName);
+  });
+}
+export function extractConfigContent(config, configPath) {
+  const extractionConfig = CONFIG_EXTRACTION_MAP[config.key];
+  
+  if (!extractionConfig || !config.metadata) {
+    return config;
+  }
+
+  const updatedConfig = JSON.parse(JSON.stringify(config)); // Deep clone
+  const { fields, fileExtension, subdirectory } = extractionConfig;
+  
+  // Create subdirectory if needed
+  const extractDir = path.join(configPath, subdirectory);
+  if (!fs.existsSync(extractDir)) {
+    fs.mkdirSync(extractDir, { recursive: true });
+    logger.debug(`Created directory: ${extractDir}`);
+  }
+
+  // Extract each field
+  for (const field of fields) {
+    if (updatedConfig.metadata[field] && typeof updatedConfig.metadata[field] === 'string') {
+      const content = updatedConfig.metadata[field];
+      
+      // Skip if content is empty or just whitespace
+      if (!content.trim()) {
+        continue;
+      }
+
+      // Generate filename - use config key if only one field, otherwise include field name
+      let fileName;
+      if (fields.length === 1) {
+        fileName = `${config.key}${fileExtension}`;
+      } else {
+        fileName = `${config.key}-${field}${fileExtension}`;
+      }
+      const filePath = path.join(extractDir, fileName);
+      const relativeRef = `./${subdirectory}/${fileName}`;
+
+      // Write content to file
+      fs.writeFileSync(filePath, content, 'utf-8');
+      logger.debug(`Extracted ${field} to: ${relativeRef}`);
+
+      // Replace with $ref
+      updatedConfig.metadata[field] = { $ref: relativeRef };
+    }
+  }
+
+  return updatedConfig;
+}
+
+/**
+ * Extract all configurations that have content to be externalized
+ * @param {Array} configs - Array of configuration objects
+ * @param {string} configPath - Path where config files are stored
+ * @returns {Array} Updated configurations with $ref objects
+ */
+export function extractAllConfigContent(configs, configPath) {
+  return configs.map(config => extractConfigContent(config, configPath));
+}
+
+/**
+ * Recursively find all $ref objects in an object
+ * @param {any} obj - Object to search
+ * @param {Array} refs - Array to collect found references (used internally)
+ * @returns {Array} Array of found $ref paths
+ */
+export function findAllRefs(obj, refs = []) {
+  if (Array.isArray(obj)) {
+    obj.forEach(item => findAllRefs(item, refs));
+  } else if (obj && typeof obj === 'object') {
+    if (obj.$ref && typeof obj.$ref === 'string') {
+      refs.push(obj.$ref);
+    } else {
+      Object.values(obj).forEach(value => findAllRefs(value, refs));
+    }
+  }
+  return refs;
+}

package/services/integrations.mjs

@@ -0,0 +1,199 @@
+import fs from 'node:fs';
+import { join } from 'node:path';
+import { logger } from './logger.mjs';
+import { extractIntegrationContent, resolveReferences } from './file-refs.mjs';
+
+const INTEGRATION_FILE_EXTENSION = '.integration.json';
+const INTEGRATIONS_API_PATH = '/api/integrations';
+const SERVER_ONLY_FIELDS = ['tenant', 'plugin', 'user', 'created', 'updated', '__v'];
+
+function slugify(value = '') {
+  return value
+    .toString()
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+function getIntegrationDisplayName(integration) {
+  return (
+    integration?.trigger?.details?.name ||
+    integration?.target?.details?.name ||
+    integration?._id ||
+    ''
+  );
+}
+
+function buildFileName(integration, index, usedNames) {
+  const preferred =
+    slugify(getIntegrationDisplayName(integration)) ||
+    slugify(integration?._id) ||
+    '';
+
+  let baseName = preferred || `integration-${index + 1}`;
+
+  while (usedNames.has(baseName)) {
+    const suffix = integration?._id ? integration._id.slice(-4) : `${usedNames.size + 1}`;
+    baseName = `${baseName}-${suffix}`;
+  }
+
+  usedNames.add(baseName);
+  return `${baseName}${INTEGRATION_FILE_EXTENSION}`;
+}
+
+function validateIntegrationPayload(data, file) {
+  if (!data?.trigger || !data?.target) {
+    throw new Error(`Integration file ${file} must include trigger and target`);
+  }
+
+  if (!data.trigger.source || !data.trigger.operation) {
+    throw new Error(`Integration ${file}: trigger must include source and operation`);
+  }
+
+  if (!data.target.source || !data.target.operation) {
+    throw new Error(`Integration ${file}: target must include source and operation`);
+  }
+}
+
+function toRequestPayload(data) {
+  return {
+    trigger: data.trigger,
+    target: data.target,
+    dataManipulation: data.dataManipulation || [],
+    active: data.active ?? false,
+  };
+}
+
+function writeIntegrationFile(filePath, integration) {
+  fs.writeFileSync(filePath, JSON.stringify(integration, null, 2), 'utf-8');
+}
+
+function sanitizeIntegrationForFile(integration) {
+  const sanitized = JSON.parse(JSON.stringify(integration));
+  SERVER_ONLY_FIELDS.forEach((field) => {
+    if (field in sanitized) {
+      delete sanitized[field];
+    }
+  });
+  return sanitized;
+}
+
+async function fetchIntegrations(sdk) {
+  return sdk.callJsonApi(INTEGRATIONS_API_PATH);
+}
+
+async function createIntegration(sdk, payload) {
+  return sdk.callJsonApi(INTEGRATIONS_API_PATH, {
+    method: 'post',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify(payload),
+  });
+}
+
+async function updateIntegration(sdk, id, payload) {
+  return sdk.callJsonApi(`${INTEGRATIONS_API_PATH}/${id}`, {
+    method: 'put',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify(payload),
+  });
+}
+
+export async function pullIntegrations(sdk, targetPath) {
+  if (!fs.existsSync(targetPath)) {
+    fs.mkdirSync(targetPath, { recursive: true });
+    logger.info(`Created directory: ${targetPath}`);
+  }
+
+  let integrations = [];
+  try {
+    integrations = await fetchIntegrations(sdk);
+  } catch (error) {
+    logger.error('Failed to fetch integrations', error);
+    throw error;
+  }
+
+  if (!Array.isArray(integrations) || integrations.length === 0) {
+    logger.warning('No integrations found to pull');
+    return;
+  }
+
+  logger.info(`Found ${integrations.length} integration(s) to pull`);
+
+  const usedNames = new Set();
+  integrations.forEach((integration, index) => {
+    const fileName = buildFileName(integration, index, usedNames);
+    const filePath = join(targetPath, fileName);
+    
+    // Extract content to files for AI agents
+    const processedIntegration = extractIntegrationContent(integration, targetPath, fileName);
+    
+    writeIntegrationFile(filePath, sanitizeIntegrationForFile(processedIntegration));
+    logger.step(`Pulled: ${getIntegrationDisplayName(integration) || integration._id || fileName}`);
+  });
+
+  logger.info(`Pulled ${integrations.length} integration(s)`);
+}
+
+export async function pushIntegrations(sdk, path, options = {}) {
+  const { targetFile } = options;
+  const directoryFiles = fs.readdirSync(path);
+  const files = targetFile ? [targetFile] : directoryFiles;
+  const integrationFiles = files.filter((f) => f.endsWith(INTEGRATION_FILE_EXTENSION));
+
+  if (integrationFiles.length === 0) {
+    if (targetFile) {
+      logger.warning(`File ${targetFile} is not an ${INTEGRATION_FILE_EXTENSION} file. Skipping.`);
+    } else {
+      logger.warning(`No integration files (*${INTEGRATION_FILE_EXTENSION}) found in ${path}`);
+    }
+    return;
+  }
+
+  logger.info(`Found ${integrationFiles.length} integration(s) to push`);
+
+  const results = [];
+
+  for (const file of integrationFiles) {
+    const filePath = join(path, file);
+    try {
+      const integrationData = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+      validateIntegrationPayload(integrationData, file);
+      
+      // Resolve any $ref references in the integration
+      const resolvedIntegration = await resolveReferences(integrationData, path);
+      
+      const payload = toRequestPayload(resolvedIntegration);
+      const displayName = getIntegrationDisplayName(resolvedIntegration) || file.replace(INTEGRATION_FILE_EXTENSION, '');
+
+      logger.step(`Pushing integration: ${displayName}`);
+
+      let response;
+      if (integrationData._id) {
+        response = await updateIntegration(sdk, integrationData._id, payload);
+        logger.success(`Updated: ${displayName}`);
+      } else {
+        response = await createIntegration(sdk, payload);
+        logger.success(`Created: ${displayName}`);
+      }
+
+      // Persist returned integration (with _id) back to disk
+      writeIntegrationFile(filePath, sanitizeIntegrationForFile(response));
+      results.push({ status: 'fulfilled' });
+    } catch (error) {
+      logger.error(`Failed to push integration file ${file}`, error);
+      results.push({ status: 'rejected', reason: error });
+    }
+  }
+
+  const failures = results.filter((result) => result.status === 'rejected');
+  if (failures.length) {
+    logger.error(`\n${failures.length} integration(s) failed to push:`);
+    failures.forEach((failure) => {
+      logger.error(`  • ${failure.reason?.message || 'Unknown error'}`);
+    });
+    throw new Error(`Failed to push ${failures.length} integration(s)`);
+  }
+
+  logger.info(`Pushed ${integrationFiles.length} integration(s)`);
+}

package/services/micro-frontends.mjs

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { logger } from './logger.mjs';
+import { loadReference } from './file-refs.mjs';
 
 /**
  * Convert a string to kebab-case
@@ -59,41 +60,6 @@ export function extractMicroFrontendStructures(microFrontends, pluginPath) {
   });
 }
 
-/**
- * Load content from a $ref reference
- * @param {string} ref - Reference path (relative, absolute, or URL)
- * @param {string} basePath - Base path for resolving relative references
- * @returns {Promise<string>} Loaded content
- */
-async function loadReference(ref, basePath) {
-  // Handle HTTP/HTTPS URLs
-  if (ref.startsWith('http://') || ref.startsWith('https://')) {
-    logger.debug(`Loading structure from URL: ${ref}`);
-    try {
-      const response = await fetch(ref);
-      if (!response.ok) {
-        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-      }
-      return await response.text();
-    } catch (error) {
-      throw new Error(`Failed to load from URL ${ref}: ${error.message}`);
-    }
-  }
-
-  // Handle absolute and relative paths
-  const filePath = path.isAbsolute(ref) 
-    ? ref 
-    : path.resolve(basePath, ref);
-
-  logger.debug(`Loading structure from file: ${filePath}`);
-  
-  if (!fs.existsSync(filePath)) {
-    throw new Error(`Referenced file does not exist: ${filePath}`);
-  }
-
-  return fs.readFileSync(filePath, 'utf-8');
-}
-
 /**
  * Resolve micro-frontend structures from $ref references
  * @param {Array} microFrontends - Array of micro-frontend objects