@qelos/plugins-cli 0.0.19 → 0.0.20

package/README.md

@@ -47,7 +47,7 @@ qplay create my-app
 
 ### Pull
 
-Pull resources from your Qelos instance to your local filesystem. This allows you to work on components, plugins, integrations, and blueprints locally.
+Pull resources from your Qelos instance to your local filesystem. This allows you to work on components, blueprints, configs, plugins, blocks, integrations, and connections locally.
 
 **Syntax:**
 ```bash
@@ -55,7 +55,7 @@ qelos pull <type> <path>
 ```
 
 **Arguments:**
-- `type` - Type of resource to pull (e.g., `components`, `plugins`, `integrations`, `blueprints`)
+- `type` - Type of resource to pull (e.g., `components`, `plugins`, `integrations`, `connections`, `blueprints`)
 - `path` - Local directory path where resources will be saved
 
 **Example - Pull Components:**
@@ -82,7 +82,7 @@ All 5 components pulled to ./my-components
 
 ### Push
 
-Push local resources to your Qelos instance. This allows you to update or create components, plugins, integrations, and blueprints from your local filesystem.
+Push local resources to your Qelos instance. This allows you to update or create components, blueprints, configs, plugins, blocks, integrations, and connections from your local filesystem.
 
 **Syntax:**
 ```bash
@@ -90,7 +90,7 @@ qelos push <type> <path>
 ```
 
 **Arguments:**
-- `type` - Type of resource to push (e.g., `components`, `plugins`, `integrations`, `blueprints`)
+- `type` - Type of resource to push (e.g., `components`, `plugins`, `integrations`, `connections`, `blueprints`)
 - `path` - Local directory path containing the resources to push
 
 **Example - Push Components:**
@@ -129,6 +129,39 @@ qelos pull components ./local-components
 qelos push components ./local-components
 ```
 
+### Integrations
+
+- Pulled integrations are stored as `.integration.json` files that exclude server-only fields such as `tenant`, `user`, `created`, `updated`, and `__v`.  
+- When pushing, the CLI automatically recalculates backend-only properties (like `kind`) so you do not need to keep them in local files.
+
+### Connections (Integration Sources)
+
+Connections (integration sources) are now fully supported via `qelos pull connections <path>` and `qelos push connections <path>`.
+
+- Each connection is stored as `<name>.connection.json` containing `name`, `kind`, `labels`, `metadata`, and an `authentication` placeholder:
+
+```json
+{
+  "_id": "64f1...",
+  "name": "OpenAI",
+  "kind": "openai",
+  "labels": ["ai"],
+  "metadata": { "defaultModel": "gpt-4o" },
+  "authentication": {
+    "$var": "INTEGRATION_AUTH_OPENAI"
+  }
+}
+```
+
+- At push time, the CLI reads the referenced environment variable (e.g., `INTEGRATION_AUTH_OPENAI`) which must contain a JSON string with the real credentials:
+
+```bash
+export INTEGRATION_AUTH_OPENAI='{"token":"sk-..."}'
+```
+
+- If the env var is missing, the CLI skips updating the secure authentication payload and only syncs metadata.
+- When new connections are created, the CLI persists the returned `_id` so future pushes update the same record.
+
 ## Help
 
 View all available commands and options:

package/commands/pull.mjs

@@ -8,7 +8,7 @@ export default function pullCommand(program) {
           .positional('type', {
             describe: 'Type of the resource to pull. Can be components, blueprints, configurations, plugins, blocks, or all.',
             type: 'string',
-            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'all', '*'],
+            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'all', '*'],
             required: true
           })
           .positional('path', {

package/commands/push.mjs

@@ -8,7 +8,7 @@ export default function createCommand(program) {
           .positional('type', {
             describe: 'Type of the resource to push. Can be components, blueprints, configurations, plugins, blocks, or all.',
             type: 'string',
-            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'all', '*'],
+            choices: ['components', 'blueprints', 'configs', 'plugins', 'blocks', 'integrations', 'all', '*'],
             required: true
           })
           .positional('path', {

package/controllers/pull.mjs

@@ -4,6 +4,8 @@ import { pullBlueprints } from '../services/blueprints.mjs';
 import { pullConfigurations } from '../services/configurations.mjs';
 import { pullPlugins } from '../services/plugins.mjs';
 import { pullBlocks } from '../services/blocks.mjs';
+import { pullIntegrations } from '../services/integrations.mjs';
+import { pullConnections } from '../services/connections.mjs';
 import { logger } from '../services/logger.mjs';
 import fs from 'node:fs';
 import path from 'node:path';
@@ -36,7 +38,9 @@ export default async function pullController({ type, path: targetPath = './' })
         { name: 'blueprints', fn: pullBlueprints },
         { name: 'configs', fn: pullConfigurations },
         { name: 'plugins', fn: pullPlugins },
-        { name: 'blocks', fn: pullBlocks }
+        { name: 'blocks', fn: pullBlocks },
+        { name: 'integrations', fn: pullIntegrations },
+        { name: 'connections', fn: pullConnections }
       ];
 
       for (const { name, fn } of types) {
@@ -64,11 +68,15 @@ export default async function pullController({ type, path: targetPath = './' })
       await pullPlugins(sdk, targetPath);
     } else if (type === 'blocks') {
       await pullBlocks(sdk, targetPath);
+    } else if (type === 'integrations' || type === 'integration') {
+      await pullIntegrations(sdk, targetPath);
+    } else if (type === 'connections' || type === 'connection') {
+      await pullConnections(sdk, targetPath);
     } else if (type === 'config' || type === 'configs' || type === 'configuration') {
       await pullConfigurations(sdk, targetPath);
     } else {
       logger.error(`Unknown type: ${type}`);
-      logger.info('Supported types: components, blueprints, plugins, blocks, config, configs, configuration, all');
+      logger.info('Supported types: components, blueprints, plugins, blocks, integrations, connections, config, configs, configuration, all');
       process.exit(1);
     }
 

package/controllers/push.mjs

@@ -4,6 +4,8 @@ import { pushBlueprints } from '../services/blueprints.mjs';
 import { pushConfigurations } from '../services/configurations.mjs';
 import { pushPlugins } from '../services/plugins.mjs';
 import { pushBlocks } from '../services/blocks.mjs';
+import { pushIntegrations } from '../services/integrations.mjs';
+import { pushConnections } from '../services/connections.mjs';
 import { logger } from '../services/logger.mjs';
 import fs from 'node:fs';
 import path from 'node:path';
@@ -45,7 +47,9 @@ export default async function pushController({ type, path: sourcePath }) {
         { name: 'blueprints', fn: pushBlueprints },
         { name: 'configs', fn: pushConfigurations },
         { name: 'plugins', fn: pushPlugins },
-        { name: 'blocks', fn: pushBlocks }
+        { name: 'blocks', fn: pushBlocks },
+        { name: 'integrations', fn: pushIntegrations },
+        { name: 'connections', fn: pushConnections }
       ];
 
       for (const { name, fn } of types) {
@@ -80,11 +84,15 @@ export default async function pushController({ type, path: sourcePath }) {
       await pushPlugins(sdk, basePath, { targetFile });
     } else if (type === 'blocks') {
       await pushBlocks(sdk, basePath, { targetFile });
-    }  else if (type === 'config' || type === 'configs' || type === 'configuration') {
+    } else if (type === 'integrations' || type === 'integration') {
+      await pushIntegrations(sdk, basePath, { targetFile });
+    } else if (type === 'connections' || type === 'connection') {
+      await pushConnections(sdk, basePath, { targetFile });
+    } else if (type === 'config' || type === 'configs' || type === 'configuration') {
       await pushConfigurations(sdk, basePath, { targetFile });
     } else {
       logger.error(`Unknown type: ${type}`);
-      logger.info('Supported types: components, blueprints, plugins, blocks, config, configs, configuration, all');
+      logger.info('Supported types: components, blueprints, plugins, blocks, integrations, connections, config, configs, configuration, all');
       process.exit(1);
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qelos/plugins-cli",
-  "version": "0.0.19",
+  "version": "0.0.20",
   "description": "CLI to manage QELOS plugins",
   "main": "cli.mjs",
   "bin": {

package/services/blueprint-sdk-guides.mjs

@@ -68,12 +68,12 @@ function buildMarkdown({ blueprint, interfaceName, entityVarName, interfaceDefin
     '',
     '### Create an Entity',
     '```ts',
-    `const created = await ${entityVarName}.create(${exampleLiteral});`,
+    `const created = await ${entityVarName}.create({\n  metadata: ${indentLiteral(exampleLiteral, 2)},\n});`,
     '```',
     '',
     '### Update an Entity',
     '```ts',
-    `const updated = await ${entityVarName}.update('replace-with-entity-id', {\n  ...${exampleLiteral.replace(/\n/g, '\n  ')},\n});`,
+    `const updated = await ${entityVarName}.update('replace-with-entity-id', {\n  metadata: {\n    ...oldMetadata,\n    ...${indentLiteral(exampleLiteral, 4)},\n  },\n});`,
     '```',
     '',
     '### Delete an Entity',
@@ -219,6 +219,11 @@ function stringifyObjectLiteral(value, level = 0) {
   return String(value);
 }
 
+function indentLiteral(literal, spaces) {
+  const indent = ' '.repeat(spaces);
+  return literal.replace(/\n/g, `\n${indent}`);
+}
+
 function toCamelCase(value) {
   return (value || '')
     .replace(/[-_\s]+(.)?/g, (_, chr) => (chr ? chr.toUpperCase() : ''))

package/services/connections.mjs

@@ -0,0 +1,239 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { logger } from './logger.mjs';
+
+const CONNECTION_FILE_EXTENSION = '.connection.json';
+const AUTH_PLACEHOLDER_KEY = '$var';
+
+function slugify(value = '') {
+  return value
+    .toString()
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+function ensureDirectory(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+    logger.info(`Created directory: ${dirPath}`);
+  }
+}
+
+function getDefaultAuthEnvVar(connection) {
+  if (connection?.authentication?.[AUTH_PLACEHOLDER_KEY]) {
+    return connection.authentication[AUTH_PLACEHOLDER_KEY];
+  }
+
+  if (connection?._id) {
+    return `INTEGRATION_AUTH_${connection._id}`;
+  }
+
+  const slug = slugify(connection?.name || 'connection').replace(/-/g, '_').toUpperCase();
+  const suffix = slug ? slug.slice(-6) : Math.random().toString(36).slice(-6).toUpperCase();
+  return `INTEGRATION_AUTH_${suffix || 'NEW'}`;
+}
+
+function buildFileName(connection, index, usedNames) {
+  const base =
+    slugify(connection?.name) ||
+    (connection?._id ? slugify(connection._id) : '') ||
+    `connection-${index + 1}`;
+
+  let fileName = base;
+  while (usedNames.has(fileName)) {
+    fileName = `${base}-${usedNames.size + 1}`;
+  }
+  usedNames.add(fileName);
+  return `${fileName}${CONNECTION_FILE_EXTENSION}`;
+}
+
+function sanitizeConnectionForFile(connection) {
+  return {
+    _id: connection._id,
+    name: connection.name,
+    kind: connection.kind,
+    labels: connection.labels || [],
+    metadata: connection.metadata || {},
+    authentication: {
+      [AUTH_PLACEHOLDER_KEY]: getDefaultAuthEnvVar(connection),
+    },
+  };
+}
+
+function writeConnectionFile(filePath, connection) {
+  fs.writeFileSync(filePath, JSON.stringify(connection, null, 2), 'utf-8');
+}
+
+function validateConnectionPayload(data, fileName) {
+  if (!data?.name) {
+    throw new Error(`Connection file ${fileName} must include a name`);
+  }
+
+  if (!data?.kind) {
+    throw new Error(`Connection file ${fileName} must include a kind`);
+  }
+
+  if (!data?.metadata || typeof data.metadata !== 'object') {
+    throw new Error(`Connection file ${fileName} must include metadata object`);
+  }
+
+  if (data.labels && !Array.isArray(data.labels)) {
+    throw new Error(`Connection file ${fileName} has invalid labels (must be an array)`);
+  }
+}
+
+function extractAuthenticationPayload(connectionFile, fileName) {
+  const authField = connectionFile.authentication;
+
+  if (!authField || typeof authField !== 'object') {
+    return { payload: undefined, envVar: null };
+  }
+
+  const envVarName = authField[AUTH_PLACEHOLDER_KEY];
+
+  if (!envVarName) {
+    throw new Error(
+      `Connection file ${fileName} must define authentication as { "${AUTH_PLACEHOLDER_KEY}": "INTEGRATION_AUTH_..." }`
+    );
+  }
+
+  const raw = process.env[envVarName];
+
+  if (!raw) {
+    if (!connectionFile._id) {
+      throw new Error(
+        `Environment variable ${envVarName} is required to create connection defined in ${fileName}`
+      );
+    }
+
+    logger.info(
+      `Skipping authentication update for ${connectionFile.name} (env ${envVarName} not set)`
+    );
+    return { payload: undefined, envVar: envVarName };
+  }
+
+  try {
+    const parsed = JSON.parse(raw);
+    if (typeof parsed !== 'object' || !parsed) {
+      throw new Error('Authentication env must contain a JSON object');
+    }
+    return { payload: parsed, envVar: envVarName };
+  } catch (error) {
+    throw new Error(
+      `Failed to parse JSON from env ${envVarName} for connection ${fileName}: ${error.message}`
+    );
+  }
+}
+
+export async function pullConnections(sdk, targetPath) {
+  ensureDirectory(targetPath);
+
+  let connections = [];
+  try {
+    connections = await sdk.integrationSources.getList();
+  } catch (error) {
+    logger.error('Failed to fetch integration sources', error);
+    throw error;
+  }
+
+  if (!Array.isArray(connections) || connections.length === 0) {
+    logger.warning('No connections found to pull');
+    return;
+  }
+
+  logger.info(`Found ${connections.length} connection(s) to pull`);
+  const usedNames = new Set();
+
+  connections.forEach((connection, index) => {
+    const fileName = buildFileName(connection, index, usedNames);
+    const filePath = path.join(targetPath, fileName);
+    writeConnectionFile(filePath, sanitizeConnectionForFile(connection));
+    logger.step(`Pulled connection: ${connection.name || connection._id}`);
+  });
+
+  logger.info(`Pulled ${connections.length} connection(s)`);
+}
+
+export async function pushConnections(sdk, sourcePath, options = {}) {
+  const { targetFile } = options;
+  const directoryFiles = fs.readdirSync(sourcePath);
+  const files = targetFile ? [targetFile] : directoryFiles;
+  const connectionFiles = files.filter((file) => file.endsWith(CONNECTION_FILE_EXTENSION));
+
+  if (connectionFiles.length === 0) {
+    if (targetFile) {
+      logger.warning(
+        `File ${targetFile} is not a ${CONNECTION_FILE_EXTENSION} connection file. Skipping.`
+      );
+    } else {
+      logger.warning(`No connection files (*${CONNECTION_FILE_EXTENSION}) found in ${sourcePath}`);
+    }
+    return;
+  }
+
+  logger.info(`Found ${connectionFiles.length} connection(s) to push`);
+
+  const results = [];
+
+  for (const file of connectionFiles) {
+    const filePath = path.join(sourcePath, file);
+
+    try {
+      const connectionData = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+      validateConnectionPayload(connectionData, file);
+
+      const { payload: authentication, envVar } = extractAuthenticationPayload(connectionData, file);
+
+      const requestBody = {
+        name: connectionData.name,
+        kind: connectionData.kind,
+        labels: connectionData.labels || [],
+        metadata: connectionData.metadata || {},
+        ...(authentication ? { authentication } : {}),
+      };
+
+      logger.step(`Pushing connection: ${connectionData.name}`);
+      let response;
+
+      if (connectionData._id) {
+        response = await sdk.integrationSources.update(connectionData._id, requestBody);
+        logger.success(`Updated connection: ${connectionData.name}`);
+      } else {
+        response = await sdk.integrationSources.create(requestBody);
+        logger.success(`Created connection: ${connectionData.name}`);
+      }
+
+      const authPlaceholder = connectionData.authentication?.[AUTH_PLACEHOLDER_KEY]
+        ? connectionData.authentication
+        : { [AUTH_PLACEHOLDER_KEY]: envVar || getDefaultAuthEnvVar(response) };
+
+      const fileContent = {
+        _id: response._id,
+        name: response.name,
+        kind: response.kind,
+        labels: response.labels || [],
+        metadata: response.metadata || {},
+        authentication: authPlaceholder,
+      };
+
+      writeConnectionFile(filePath, fileContent);
+      results.push({ status: 'fulfilled' });
+    } catch (error) {
+      logger.error(`Failed to push connection file ${file}`, error);
+      results.push({ status: 'rejected', reason: error });
+    }
+  }
+
+  const failures = results.filter((result) => result.status === 'rejected');
+  if (failures.length) {
+    logger.error(`\n${failures.length} connection(s) failed to push:`);
+    failures.forEach((failure) => {
+      logger.error(`  • ${failure.reason?.message || 'Unknown error'}`);
+    });
+    throw new Error(`Failed to push ${failures.length} connection(s)`);
+  }
+
+  logger.info(`Pushed ${connectionFiles.length} connection(s)`);
+}

package/services/integrations.mjs

@@ -0,0 +1,190 @@
+import fs from 'node:fs';
+import { join } from 'node:path';
+import { logger } from './logger.mjs';
+
+const INTEGRATION_FILE_EXTENSION = '.integration.json';
+const INTEGRATIONS_API_PATH = '/api/integrations';
+const SERVER_ONLY_FIELDS = ['tenant', 'plugin', 'user', 'created', 'updated', '__v'];
+
+function slugify(value = '') {
+  return value
+    .toString()
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+function getIntegrationDisplayName(integration) {
+  return (
+    integration?.trigger?.details?.name ||
+    integration?.target?.details?.name ||
+    integration?._id ||
+    ''
+  );
+}
+
+function buildFileName(integration, index, usedNames) {
+  const preferred =
+    slugify(getIntegrationDisplayName(integration)) ||
+    slugify(integration?._id) ||
+    '';
+
+  let baseName = preferred || `integration-${index + 1}`;
+
+  while (usedNames.has(baseName)) {
+    const suffix = integration?._id ? integration._id.slice(-4) : `${usedNames.size + 1}`;
+    baseName = `${baseName}-${suffix}`;
+  }
+
+  usedNames.add(baseName);
+  return `${baseName}${INTEGRATION_FILE_EXTENSION}`;
+}
+
+function validateIntegrationPayload(data, file) {
+  if (!data?.trigger || !data?.target) {
+    throw new Error(`Integration file ${file} must include trigger and target`);
+  }
+
+  if (!data.trigger.source || !data.trigger.operation) {
+    throw new Error(`Integration ${file}: trigger must include source and operation`);
+  }
+
+  if (!data.target.source || !data.target.operation) {
+    throw new Error(`Integration ${file}: target must include source and operation`);
+  }
+}
+
+function toRequestPayload(data) {
+  return {
+    trigger: data.trigger,
+    target: data.target,
+    dataManipulation: data.dataManipulation || [],
+    active: data.active ?? false,
+  };
+}
+
+function writeIntegrationFile(filePath, integration) {
+  fs.writeFileSync(filePath, JSON.stringify(integration, null, 2), 'utf-8');
+}
+
+function sanitizeIntegrationForFile(integration) {
+  const sanitized = JSON.parse(JSON.stringify(integration));
+  SERVER_ONLY_FIELDS.forEach((field) => {
+    if (field in sanitized) {
+      delete sanitized[field];
+    }
+  });
+  return sanitized;
+}
+
+async function fetchIntegrations(sdk) {
+  return sdk.callJsonApi(INTEGRATIONS_API_PATH);
+}
+
+async function createIntegration(sdk, payload) {
+  return sdk.callJsonApi(INTEGRATIONS_API_PATH, {
+    method: 'post',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify(payload),
+  });
+}
+
+async function updateIntegration(sdk, id, payload) {
+  return sdk.callJsonApi(`${INTEGRATIONS_API_PATH}/${id}`, {
+    method: 'put',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify(payload),
+  });
+}
+
+export async function pullIntegrations(sdk, targetPath) {
+  if (!fs.existsSync(targetPath)) {
+    fs.mkdirSync(targetPath, { recursive: true });
+    logger.info(`Created directory: ${targetPath}`);
+  }
+
+  let integrations = [];
+  try {
+    integrations = await fetchIntegrations(sdk);
+  } catch (error) {
+    logger.error('Failed to fetch integrations', error);
+    throw error;
+  }
+
+  if (!Array.isArray(integrations) || integrations.length === 0) {
+    logger.warning('No integrations found to pull');
+    return;
+  }
+
+  logger.info(`Found ${integrations.length} integration(s) to pull`);
+
+  const usedNames = new Set();
+  integrations.forEach((integration, index) => {
+    const fileName = buildFileName(integration, index, usedNames);
+    const filePath = join(targetPath, fileName);
+    writeIntegrationFile(filePath, sanitizeIntegrationForFile(integration));
+    logger.step(`Pulled: ${getIntegrationDisplayName(integration) || integration._id || fileName}`);
+  });
+
+  logger.info(`Pulled ${integrations.length} integration(s)`);
+}
+
+export async function pushIntegrations(sdk, path, options = {}) {
+  const { targetFile } = options;
+  const directoryFiles = fs.readdirSync(path);
+  const files = targetFile ? [targetFile] : directoryFiles;
+  const integrationFiles = files.filter((f) => f.endsWith(INTEGRATION_FILE_EXTENSION));
+
+  if (integrationFiles.length === 0) {
+    if (targetFile) {
+      logger.warning(`File ${targetFile} is not an ${INTEGRATION_FILE_EXTENSION} file. Skipping.`);
+    } else {
+      logger.warning(`No integration files (*${INTEGRATION_FILE_EXTENSION}) found in ${path}`);
+    }
+    return;
+  }
+
+  logger.info(`Found ${integrationFiles.length} integration(s) to push`);
+
+  const results = [];
+
+  for (const file of integrationFiles) {
+    const filePath = join(path, file);
+    try {
+      const integrationData = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+      validateIntegrationPayload(integrationData, file);
+      const payload = toRequestPayload(integrationData);
+      const displayName = getIntegrationDisplayName(integrationData) || file.replace(INTEGRATION_FILE_EXTENSION, '');
+
+      logger.step(`Pushing integration: ${displayName}`);
+
+      let response;
+      if (integrationData._id) {
+        response = await updateIntegration(sdk, integrationData._id, payload);
+        logger.success(`Updated: ${displayName}`);
+      } else {
+        response = await createIntegration(sdk, payload);
+        logger.success(`Created: ${displayName}`);
+      }
+
+      // Persist returned integration (with _id) back to disk
+      writeIntegrationFile(filePath, sanitizeIntegrationForFile(response));
+      results.push({ status: 'fulfilled' });
+    } catch (error) {
+      logger.error(`Failed to push integration file ${file}`, error);
+      results.push({ status: 'rejected', reason: error });
+    }
+  }
+
+  const failures = results.filter((result) => result.status === 'rejected');
+  if (failures.length) {
+    logger.error(`\n${failures.length} integration(s) failed to push:`);
+    failures.forEach((failure) => {
+      logger.error(`  • ${failure.reason?.message || 'Unknown error'}`);
+    });
+    throw new Error(`Failed to push ${failures.length} integration(s)`);
+  }
+
+  logger.info(`Pushed ${integrationFiles.length} integration(s)`);
+}