newo 3.1.0 → 3.2.0

package/dist/sync/integrations.js

@@ -0,0 +1,340 @@
+/**
+ * Integration and connector synchronization module
+ * Handles pull/push of integrations and connectors to/from NEWO platform
+ */
+import path from 'path';
+import fs from 'fs-extra';
+import yaml from 'js-yaml';
+import { listIntegrations, listConnectors, getIntegrationSettings, createConnector, updateConnector, deleteConnector, listOutgoingWebhooks, listIncomingWebhooks } from '../api.js';
+/**
+ * Pull all integrations and connectors from NEWO platform
+ */
+export async function pullIntegrations(client, customerDir, verbose = false) {
+    if (verbose)
+        console.log('\n📦 Pulling integrations from NEWO platform...\n');
+    // Create integrations directory
+    const integrationsDir = path.join(customerDir, 'integrations');
+    await fs.ensureDir(integrationsDir);
+    // Fetch all integrations
+    const integrations = await listIntegrations(client);
+    if (verbose)
+        console.log(`✓ Found ${integrations.length} integrations`);
+    const integrationsMetadata = [];
+    // Process each integration
+    for (const integration of integrations) {
+        if (verbose)
+            console.log(`\n  📦 Processing: ${integration.title} (${integration.idn})`);
+        // Add to metadata list
+        integrationsMetadata.push({
+            id: integration.id,
+            idn: integration.idn,
+            title: integration.title,
+            description: integration.description,
+            channel: integration.channel,
+            is_disabled: integration.is_disabled
+        });
+        // Create integration directory
+        const integrationDir = path.join(integrationsDir, integration.idn);
+        await fs.ensureDir(integrationDir);
+        // Fetch integration settings
+        let integrationSettings = [];
+        try {
+            integrationSettings = await getIntegrationSettings(client, integration.id);
+        }
+        catch (error) {
+            // Settings endpoint may not be available for all integrations
+            if (verbose && error.response?.status !== 404) {
+                console.log(`     ⚠ Could not fetch settings: ${error.message}`);
+            }
+        }
+        // Save combined integration file (metadata + settings)
+        const integrationFile = path.join(integrationDir, `${integration.idn}.yaml`);
+        const integrationData = {
+            id: integration.id,
+            idn: integration.idn,
+            title: integration.title,
+            description: integration.description,
+            channel: integration.channel,
+            is_disabled: integration.is_disabled
+        };
+        // Add settings array if any settings exist
+        if (integrationSettings.length > 0) {
+            integrationData.settings = integrationSettings;
+        }
+        await fs.writeFile(integrationFile, yaml.dump(integrationData, { lineWidth: -1 }));
+        if (verbose)
+            console.log(`     ✓ Saved integration → ${integration.idn}.yaml (${integrationSettings.length} settings)`);
+        // Fetch and save connectors
+        const connectors = await listConnectors(client, integration.id);
+        if (verbose)
+            console.log(`     Connectors: ${connectors.length} found`);
+        if (connectors.length > 0) {
+            const connectorsDir = path.join(integrationDir, 'connectors');
+            await fs.ensureDir(connectorsDir);
+            for (const connector of connectors) {
+                const connectorMetadata = {
+                    id: connector.id,
+                    connector_idn: connector.connector_idn,
+                    title: connector.title,
+                    status: connector.status,
+                    integration_idn: integration.idn,
+                    settings: connector.settings
+                };
+                // Create subdirectory for this connector
+                const connectorDir = path.join(connectorsDir, connector.connector_idn);
+                await fs.ensureDir(connectorDir);
+                // Save connector YAML file inside its subdirectory
+                const connectorFile = path.join(connectorDir, `${connector.connector_idn}.yaml`);
+                await fs.writeFile(connectorFile, yaml.dump(connectorMetadata, { lineWidth: -1 }));
+                if (verbose)
+                    console.log(`     ✓ Saved: ${connector.title} → connectors/${connector.connector_idn}/${connector.connector_idn}.yaml`);
+            }
+        }
+    }
+    // Fetch and save webhooks (for API integration connectors only)
+    if (verbose)
+        console.log(`\n📡 Fetching webhooks...`);
+    try {
+        const outgoingWebhooks = await listOutgoingWebhooks(client);
+        const incomingWebhooks = await listIncomingWebhooks(client);
+        if (verbose)
+            console.log(`✓ Found ${outgoingWebhooks.length} outgoing webhooks`);
+        if (verbose)
+            console.log(`✓ Found ${incomingWebhooks.length} incoming webhooks`);
+        // Group webhooks by connector_idn
+        const outgoingByConnector = new Map();
+        const incomingByConnector = new Map();
+        outgoingWebhooks.forEach(webhook => {
+            if (!outgoingByConnector.has(webhook.connector_idn)) {
+                outgoingByConnector.set(webhook.connector_idn, []);
+            }
+            outgoingByConnector.get(webhook.connector_idn).push(webhook);
+        });
+        incomingWebhooks.forEach(webhook => {
+            if (!incomingByConnector.has(webhook.connector_idn)) {
+                incomingByConnector.set(webhook.connector_idn, []);
+            }
+            incomingByConnector.get(webhook.connector_idn).push(webhook);
+        });
+        // Save webhooks to appropriate connector directories
+        for (const integration of integrations) {
+            const integrationDir = path.join(integrationsDir, integration.idn);
+            const connectorsDir = path.join(integrationDir, 'connectors');
+            if (await fs.pathExists(connectorsDir)) {
+                const connectors = await listConnectors(client, integration.id);
+                for (const connector of connectors) {
+                    const connectorWebhooksDir = path.join(connectorsDir, connector.connector_idn, 'webhooks');
+                    const outgoing = outgoingByConnector.get(connector.connector_idn) || [];
+                    const incoming = incomingByConnector.get(connector.connector_idn) || [];
+                    if (outgoing.length > 0 || incoming.length > 0) {
+                        await fs.ensureDir(connectorWebhooksDir);
+                        if (outgoing.length > 0) {
+                            const outgoingFile = path.join(connectorWebhooksDir, 'outgoing.yaml');
+                            await fs.writeFile(outgoingFile, yaml.dump({ webhooks: outgoing }, { lineWidth: -1 }));
+                            if (verbose)
+                                console.log(`     ✓ Saved: ${outgoing.length} outgoing webhooks → ${connector.connector_idn}/webhooks/outgoing.yaml`);
+                        }
+                        if (incoming.length > 0) {
+                            const incomingFile = path.join(connectorWebhooksDir, 'incoming.yaml');
+                            await fs.writeFile(incomingFile, yaml.dump({ webhooks: incoming }, { lineWidth: -1 }));
+                            if (verbose)
+                                console.log(`     ✓ Saved: ${incoming.length} incoming webhooks → ${connector.connector_idn}/webhooks/incoming.yaml`);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    catch (error) {
+        if (verbose)
+            console.log(`⚠ Could not fetch webhooks: ${error.message}`);
+    }
+    // Save master integrations list
+    const integrationsData = { integrations: integrationsMetadata };
+    const integrationsFile = path.join(integrationsDir, 'integrations.yaml');
+    await fs.writeFile(integrationsFile, yaml.dump(integrationsData, { lineWidth: -1 }));
+    if (verbose)
+        console.log(`\n✅ Saved ${integrations.length} integrations to integrations/integrations.yaml\n`);
+}
+/**
+ * Push integration changes from local files to NEWO platform
+ */
+export async function pushIntegrations(client, customerDir, verbose = false) {
+    if (verbose)
+        console.log('\n📤 Pushing integration changes to NEWO platform...\n');
+    const integrationsDir = path.join(customerDir, 'integrations');
+    // Check if integrations directory exists
+    if (!await fs.pathExists(integrationsDir)) {
+        if (verbose)
+            console.log('⚠ No integrations directory found. Run pull-integrations first.');
+        return;
+    }
+    // Load remote integrations for ID mapping
+    const remoteIntegrations = await listIntegrations(client);
+    const integrationMap = new Map(); // idn -> id
+    remoteIntegrations.forEach(int => integrationMap.set(int.idn, int.id));
+    let updatedCount = 0;
+    let createdCount = 0;
+    let deletedCount = 0;
+    // Read integrations directory
+    const integrationFolders = await fs.readdir(integrationsDir);
+    for (const folder of integrationFolders) {
+        if (folder === 'integrations.yaml')
+            continue; // Skip master file
+        const integrationDir = path.join(integrationsDir, folder);
+        const stat = await fs.stat(integrationDir);
+        if (!stat.isDirectory())
+            continue;
+        const integrationIdn = folder;
+        const integrationId = integrationMap.get(integrationIdn);
+        if (!integrationId) {
+            if (verbose)
+                console.log(`⚠ Integration ${integrationIdn} not found on platform, skipping...`);
+            continue;
+        }
+        if (verbose)
+            console.log(`\n  📦 Processing: ${integrationIdn}`);
+        // Process connectors
+        const connectorsDir = path.join(integrationDir, 'connectors');
+        if (await fs.pathExists(connectorsDir)) {
+            // Load remote connectors for comparison
+            const remoteConnectors = await listConnectors(client, integrationId);
+            const remoteConnectorMap = new Map();
+            remoteConnectors.forEach(conn => remoteConnectorMap.set(conn.connector_idn, conn));
+            // Read connector subdirectories
+            const connectorDirs = await fs.readdir(connectorsDir);
+            const localConnectorIdns = new Set();
+            for (const connectorDirName of connectorDirs) {
+                const connectorPath = path.join(connectorsDir, connectorDirName);
+                const stat = await fs.stat(connectorPath);
+                if (!stat.isDirectory())
+                    continue; // Skip non-directories
+                // Read connector YAML file from within the subdirectory
+                const connectorFile = path.join(connectorPath, `${connectorDirName}.yaml`);
+                if (!await fs.pathExists(connectorFile)) {
+                    if (verbose)
+                        console.log(`     ⚠ No YAML file found in ${connectorDirName}/, skipping...`);
+                    continue;
+                }
+                const connectorData = yaml.load(await fs.readFile(connectorFile, 'utf-8'));
+                localConnectorIdns.add(connectorData.connector_idn);
+                const remoteConnector = remoteConnectorMap.get(connectorData.connector_idn);
+                if (!remoteConnector) {
+                    // Create new connector
+                    if (verbose)
+                        console.log(`     ➕ Creating connector: ${connectorData.title}`);
+                    try {
+                        await createConnector(client, integrationId, {
+                            title: connectorData.title,
+                            connector_idn: connectorData.connector_idn,
+                            integration_idn: integrationIdn,
+                            settings: connectorData.settings
+                        });
+                        createdCount++;
+                        if (verbose)
+                            console.log(`     ✅ Created: ${connectorData.title}`);
+                    }
+                    catch (error) {
+                        console.error(`     ❌ Failed to create connector: ${error.message}`);
+                    }
+                }
+                else {
+                    // Check if connector needs update
+                    const needsUpdate = hasConnectorChanged(remoteConnector, connectorData);
+                    if (needsUpdate) {
+                        if (verbose)
+                            console.log(`     🔄 Updating connector: ${connectorData.title}`);
+                        try {
+                            await updateConnector(client, remoteConnector.id, {
+                                title: connectorData.title,
+                                status: connectorData.status,
+                                settings: connectorData.settings
+                            });
+                            updatedCount++;
+                            if (verbose)
+                                console.log(`     ✅ Updated: ${connectorData.title}`);
+                        }
+                        catch (error) {
+                            console.error(`     ❌ Failed to update connector: ${error.message}`);
+                        }
+                    }
+                    else {
+                        if (verbose)
+                            console.log(`     ✓ No changes: ${connectorData.title}`);
+                    }
+                }
+            }
+            // Delete connectors that exist remotely but not locally
+            for (const [connectorIdn, remoteConnector] of remoteConnectorMap) {
+                if (!localConnectorIdns.has(connectorIdn)) {
+                    if (verbose)
+                        console.log(`     🗑️ Deleting connector: ${remoteConnector.title}`);
+                    try {
+                        await deleteConnector(client, remoteConnector.id);
+                        deletedCount++;
+                        if (verbose)
+                            console.log(`     ✅ Deleted: ${remoteConnector.title}`);
+                    }
+                    catch (error) {
+                        console.error(`     ❌ Failed to delete connector: ${error.message}`);
+                    }
+                }
+            }
+        }
+    }
+    // Always show summary if changes were made, not just in verbose mode
+    if (createdCount > 0 || updatedCount > 0 || deletedCount > 0) {
+        console.log(`\n✅ Integration push completed:`);
+        console.log(`   Created: ${createdCount} connector(s)`);
+        console.log(`   Updated: ${updatedCount} connector(s)`);
+        console.log(`   Deleted: ${deletedCount} connector(s)`);
+    }
+    else {
+        console.log(`\n✓ No connector changes to push`);
+    }
+}
+/**
+ * Check if connector has changed compared to remote version
+ */
+function hasConnectorChanged(remote, local) {
+    // Check title
+    if (remote.title !== local.title)
+        return true;
+    // Check status
+    if (remote.status !== local.status)
+        return true;
+    // Check settings
+    if (remote.settings.length !== local.settings.length)
+        return true;
+    // Compare each setting
+    const remoteSettingsMap = new Map();
+    remote.settings.forEach(s => remoteSettingsMap.set(s.idn, s.value));
+    for (const localSetting of local.settings) {
+        const remoteValue = remoteSettingsMap.get(localSetting.idn);
+        if (remoteValue !== localSetting.value)
+            return true;
+    }
+    return false;
+}
+/**
+ * List all local integrations from file system
+ */
+export async function listLocalIntegrations(customerDir) {
+    const integrationsFile = path.join(customerDir, 'integrations', 'integrations.yaml');
+    if (!await fs.pathExists(integrationsFile)) {
+        return [];
+    }
+    const data = yaml.load(await fs.readFile(integrationsFile, 'utf-8'));
+    return data.integrations;
+}
+/**
+ * Get connector details from local file
+ */
+export async function getLocalConnector(customerDir, integrationIdn, connectorIdn) {
+    const connectorFile = path.join(customerDir, 'integrations', integrationIdn, 'connectors', connectorIdn, `${connectorIdn}.yaml`);
+    if (!await fs.pathExists(connectorFile)) {
+        return null;
+    }
+    return yaml.load(await fs.readFile(connectorFile, 'utf-8'));
+}
+//# sourceMappingURL=integrations.js.map

package/dist/sync/projects.js

@@ -2,7 +2,7 @@
  * Project synchronization operations
  */
 import { listProjects, listAgents, listFlowSkills, listFlowEvents, listFlowStates } from '../api.js';
-import { ensureState, writeFileSafe, mapPath, projectMetadataPath, agentMetadataPath, flowMetadataPath, skillMetadataPath, skillScriptPath, skillFolderPath, flowsYamlPath, customerAttributesPath } from '../fsutil.js';
+import { ensureState, writeFileSafe, mapPath, projectMetadataPath, agentMetadataPath, flowMetadataPath, skillMetadataPath, skillScriptPath, skillFolderPath, flowsYamlPath, customerAttributesPath, customerProjectsDir, projectDir } from '../fsutil.js';
 import { findSkillScriptFiles, isContentDifferent, askForOverwrite, getExtensionForRunner } from './skill-files.js';
 import fs from 'fs-extra';
 import { sha256, saveHashes } from '../hash.js';
@@ -16,6 +16,174 @@ export function isProjectMap(x) {
 export function isLegacyProjectMap(x) {
     return typeof x === 'object' && x !== null && 'projectId' in x && 'agents' in x;
 }
+/**
+ * Ask user for deletion confirmation
+ */
+async function askForDeletion(entityType, entityPath) {
+    const readline = await import('readline');
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout
+    });
+    const answer = await new Promise((resolve) => {
+        rl.question(`\n🗑️  Delete ${entityType}: ${entityPath}? (y)es/(n)o/(a)ll/(q)uit: `, resolve);
+    });
+    rl.close();
+    const choice = answer.toLowerCase().trim();
+    if (choice === 'q' || choice === 'quit') {
+        return 'quit';
+    }
+    if (choice === 'a' || choice === 'all') {
+        return 'all';
+    }
+    if (choice === 'y' || choice === 'yes') {
+        return 'yes';
+    }
+    return 'no';
+}
+/**
+ * Clean up deleted entities (projects, agents, flows, skills) that no longer exist remotely
+ */
+async function cleanupDeletedEntities(customerIdn, projectMap, verbose = false) {
+    const projectsDir = customerProjectsDir(customerIdn);
+    if (!(await fs.pathExists(projectsDir))) {
+        return;
+    }
+    const deletedEntities = [];
+    // Scan local filesystem for entities
+    const localProjects = await fs.readdir(projectsDir);
+    for (const projectIdn of localProjects) {
+        const projectPath = projectDir(customerIdn, projectIdn);
+        const projectStat = await fs.stat(projectPath).catch(() => null);
+        // Skip files
+        if (!projectStat || !projectStat.isDirectory())
+            continue;
+        // Skip flows.yaml
+        if (projectIdn === 'flows.yaml')
+            continue;
+        // Check if project exists in map
+        const projectData = projectMap.projects[projectIdn];
+        if (!projectData) {
+            // Entire project was deleted remotely
+            deletedEntities.push({
+                type: 'project',
+                path: projectPath,
+                displayPath: projectIdn
+            });
+            continue;
+        }
+        // Scan for agents within this project
+        try {
+            const localAgents = await fs.readdir(projectPath);
+            for (const agentIdn of localAgents) {
+                const agentPath = `${projectPath}/${agentIdn}`;
+                const agentStat = await fs.stat(agentPath).catch(() => null);
+                // Skip files and metadata.yaml
+                if (!agentStat || !agentStat.isDirectory())
+                    continue;
+                // Check if agent exists in map
+                const agentData = projectData.agents[agentIdn];
+                if (!agentData) {
+                    // Agent was deleted remotely
+                    deletedEntities.push({
+                        type: 'agent',
+                        path: agentPath,
+                        displayPath: `${projectIdn}/${agentIdn}`
+                    });
+                    continue;
+                }
+                // Scan for flows within this agent
+                try {
+                    const localFlows = await fs.readdir(agentPath);
+                    for (const flowIdn of localFlows) {
+                        const flowPath = `${agentPath}/${flowIdn}`;
+                        const flowStat = await fs.stat(flowPath).catch(() => null);
+                        // Skip files and metadata.yaml
+                        if (!flowStat || !flowStat.isDirectory())
+                            continue;
+                        // Check if flow exists in map
+                        const flowData = agentData.flows[flowIdn];
+                        if (!flowData) {
+                            // Flow was deleted remotely
+                            deletedEntities.push({
+                                type: 'flow',
+                                path: flowPath,
+                                displayPath: `${projectIdn}/${agentIdn}/${flowIdn}`
+                            });
+                            continue;
+                        }
+                        // Scan for skills within this flow
+                        try {
+                            const localSkills = await fs.readdir(flowPath);
+                            for (const skillIdn of localSkills) {
+                                const skillPath = `${flowPath}/${skillIdn}`;
+                                const skillStat = await fs.stat(skillPath).catch(() => null);
+                                // Skip files and metadata.yaml
+                                if (!skillStat || !skillStat.isDirectory())
+                                    continue;
+                                // Check if skill exists in map
+                                const skillData = flowData.skills[skillIdn];
+                                if (!skillData) {
+                                    // Skill was deleted remotely
+                                    deletedEntities.push({
+                                        type: 'skill',
+                                        path: skillPath,
+                                        displayPath: `${projectIdn}/${agentIdn}/${flowIdn}/${skillIdn}`
+                                    });
+                                }
+                            }
+                        }
+                        catch (error) {
+                            // Ignore errors reading flow directory
+                        }
+                    }
+                }
+                catch (error) {
+                    // Ignore errors reading agent directory
+                }
+            }
+        }
+        catch (error) {
+            // Ignore errors reading project directory
+        }
+    }
+    if (deletedEntities.length === 0) {
+        if (verbose)
+            console.log('✅ No deleted entities found');
+        return;
+    }
+    console.log(`\n🔍 Found ${deletedEntities.length} entity(ies) that no longer exist remotely:`);
+    for (const entity of deletedEntities) {
+        console.log(`   ${entity.type.padEnd(8)}: ${entity.displayPath}`);
+    }
+    console.log('\nThese entities will be deleted from your local filesystem.');
+    let globalDeleteAll = false;
+    for (const entity of deletedEntities) {
+        let shouldDelete = globalDeleteAll;
+        if (!globalDeleteAll) {
+            const choice = await askForDeletion(entity.type, entity.displayPath);
+            if (choice === 'quit') {
+                console.log('❌ Deletion cancelled by user');
+                return;
+            }
+            else if (choice === 'all') {
+                globalDeleteAll = true;
+                shouldDelete = true;
+            }
+            else if (choice === 'yes') {
+                shouldDelete = true;
+            }
+        }
+        if (shouldDelete) {
+            await fs.remove(entity.path);
+            console.log(`🗑️  Deleted: ${entity.displayPath}`);
+        }
+        else {
+            console.log(`⏭️  Skipped: ${entity.displayPath}`);
+        }
+    }
+    console.log(`\n✅ Cleanup completed`);
+}
 /**
  * Pull a single project and all its data
  */
@@ -284,6 +452,8 @@ export async function pullSingleProject(client, customer, projectId, verbose = f
     newHashes[flowsYamlFilePath] = sha256(flowsYamlContent);
     // Save hashes (now including flows.yaml and attributes.yaml)
     await saveHashes(newHashes, customer.idn);
+    // Detect and clean up deleted entities
+    await cleanupDeletedEntities(customer.idn, existingMap, verbose);
 }
 /**
  * Pull all projects for a customer

package/dist/sync/push.js

@@ -10,6 +10,7 @@ import yaml from 'js-yaml';
 import { generateFlowsYaml } from './metadata.js';
 import { isProjectMap, isLegacyProjectMap } from './projects.js';
 import { flowsYamlPath } from '../fsutil.js';
+import { pushAllProjectAttributes } from './attributes.js';
 /**
  * Scan filesystem for local-only entities not in the project map yet
  */
@@ -445,6 +446,20 @@ export async function pushChanged(client, customer, verbose = false, shouldPubli
     }
     if (verbose)
         console.log(`🔄 Scanned ${scanned} files, found ${pushed} changes`);
+    // Push project attributes for all projects
+    const projectsInfoMap = {};
+    for (const [projectIdn, projectData] of Object.entries(projects)) {
+        if (projectIdn && projectData.projectId) {
+            projectsInfoMap[projectIdn] = {
+                projectId: projectData.projectId,
+                projectIdn: projectData.projectIdn || projectIdn
+            };
+        }
+    }
+    const attributesUpdated = await pushAllProjectAttributes(client, customer, projectsInfoMap, verbose);
+    if (attributesUpdated > 0) {
+        pushed += attributesUpdated;
+    }
     // Regenerate flows.yaml if metadata was changed
     if (metadataChanged) {
         if (verbose)

package/dist/sync/skill-files.js

@@ -63,7 +63,7 @@ export async function validateSkillFolder(customerIdn, projectIdn, agentIdn, flo
     const warnings = [];
     const errors = [];
     if (files.length === 0) {
-        errors.push(`No script files found in skill folder: ${skillIdn}`);
+        errors.push(`No script files found in skill folder: ${folderPath}`);
     }
     else if (files.length > 1) {
         errors.push(`Multiple script files found in skill ${skillIdn}: ${files.map(f => f.fileName).join(', ')}`);

package/dist/sync/status.js

@@ -238,7 +238,8 @@ export async function status(customer, verbose = false) {
                             console.warn(`   Status check skipped - please keep only one script file.`);
                         }
                         else if (validation.files.length === 0) {
-                            console.log(`D  ${skillIdn}/ (no script files)`);
+                            const displayPath = projectIdn ? `${projectIdn}/${agentIdn}/${flowIdn}/${skillIdn}` : `${agentIdn}/${flowIdn}/${skillIdn}`;
+                            console.log(`D  ${displayPath}/ (no script files)`);
                             dirty++;
                         }
                         continue;
@@ -246,7 +247,8 @@ export async function status(customer, verbose = false) {
                     // Get the single valid script file
                     const skillFile = await getSingleSkillFile(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn);
                     if (!skillFile) {
-                        console.log(`D  ${skillIdn}/ (no valid script file)`);
+                        const displayPath = projectIdn ? `${projectIdn}/${agentIdn}/${flowIdn}/${skillIdn}` : `${agentIdn}/${flowIdn}/${skillIdn}`;
+                        console.log(`D  ${displayPath}/ (no valid script file)`);
                         dirty++;
                         if (verbose)
                             console.log(`      ❌ No valid script file found: ${skillIdn}`);