newo 2.0.5 → 3.0.0

package/dist/sync/push.js

@@ -1,19 +1,131 @@
 /**
  * Push operations for changed files
  */
-import { updateSkill } from '../api.js';
-import { ensureState, mapPath, skillMetadataPath } from '../fsutil.js';
-import { validateSkillFolder, getSingleSkillFile } from './skill-files.js';
+import { updateSkill, createAgent, createFlow, createSkill, publishFlow } from '../api.js';
+import { ensureState, mapPath, skillMetadataPath, projectDir, agentMetadataPath } from '../fsutil.js';
+import { validateSkillFolder, getSingleSkillFile, getExtensionForRunner } from './skill-files.js';
 import fs from 'fs-extra';
 import { sha256, loadHashes, saveHashes } from '../hash.js';
 import yaml from 'js-yaml';
 import { generateFlowsYaml } from './metadata.js';
 import { isProjectMap, isLegacyProjectMap } from './projects.js';
 import { flowsYamlPath } from '../fsutil.js';
+/**
+ * Scan filesystem for local-only entities not in the project map yet
+ */
+async function scanForLocalOnlyEntities(customer, projects, verbose = false) {
+    const localEntities = [];
+    let agentCount = 0;
+    let flowCount = 0;
+    let skillCount = 0;
+    // Scan each project directory
+    for (const [projectIdn] of Object.entries(projects)) {
+        const projDir = projectDir(customer.idn, projectIdn);
+        if (!(await fs.pathExists(projDir)))
+            continue;
+        if (verbose)
+            console.log(`🔍 Scanning project directory: ${projDir}`);
+        // Get all subdirectories in the project (these should be agents)
+        const agentDirs = await fs.readdir(projDir);
+        for (const agentIdn of agentDirs) {
+            const agentPath = `${projDir}/${agentIdn}`;
+            const agentStat = await fs.stat(agentPath);
+            // Skip files, only process directories
+            if (!agentStat.isDirectory())
+                continue;
+            // Skip if it's not really an agent directory (no metadata.yaml)
+            const agentMetaPath = agentMetadataPath(customer.idn, projectIdn, agentIdn);
+            if (!(await fs.pathExists(agentMetaPath)))
+                continue;
+            // Check if this agent is already in the project map
+            const projectData = projects[projectIdn];
+            if (!projectData?.agents[agentIdn]) {
+                // This is a local-only agent!
+                localEntities.push({
+                    type: 'agent',
+                    path: agentMetaPath,
+                    idn: agentIdn,
+                    projectIdn
+                });
+                agentCount++;
+                if (verbose)
+                    console.log(`  🆕 Found local-only agent: ${agentIdn}`);
+            }
+            // Now scan for flows within this agent (regardless of whether agent is local-only or not)
+            try {
+                const flowDirs = await fs.readdir(agentPath);
+                for (const flowIdn of flowDirs) {
+                    const flowPath = `${agentPath}/${flowIdn}`;
+                    const flowStat = await fs.stat(flowPath);
+                    // Skip files, only process directories
+                    if (!flowStat.isDirectory())
+                        continue;
+                    // Skip if it's not really a flow directory (no metadata.yaml)
+                    const flowMetaPath = `${flowPath}/metadata.yaml`;
+                    if (!(await fs.pathExists(flowMetaPath)))
+                        continue;
+                    // Check if this flow exists in the project map
+                    const agentData = projectData?.agents[agentIdn];
+                    if (!agentData?.flows[flowIdn]) {
+                        // This is a local-only flow!
+                        localEntities.push({
+                            type: 'flow',
+                            path: flowMetaPath,
+                            idn: flowIdn,
+                            projectIdn,
+                            agentIdn
+                        });
+                        flowCount++;
+                        if (verbose)
+                            console.log(`    🆕 Found local-only flow: ${agentIdn}/${flowIdn}`);
+                    }
+                    // Now scan for skills within this flow (regardless of whether flow is local-only or not)
+                    try {
+                        const skillDirs = await fs.readdir(flowPath);
+                        for (const skillIdn of skillDirs) {
+                            const skillPath = `${flowPath}/${skillIdn}`;
+                            const skillStat = await fs.stat(skillPath);
+                            // Skip files, only process directories
+                            if (!skillStat.isDirectory())
+                                continue;
+                            // Skip if it's not really a skill directory (no metadata.yaml)
+                            const skillMetaPath = `${skillPath}/metadata.yaml`;
+                            if (!(await fs.pathExists(skillMetaPath)))
+                                continue;
+                            // Check if this skill exists in the project map
+                            const flowData = agentData?.flows[flowIdn];
+                            if (!flowData?.skills[skillIdn]) {
+                                // This is a local-only skill!
+                                localEntities.push({
+                                    type: 'skill',
+                                    path: skillMetaPath,
+                                    idn: skillIdn,
+                                    projectIdn,
+                                    agentIdn,
+                                    flowIdn
+                                });
+                                skillCount++;
+                                if (verbose)
+                                    console.log(`      🆕 Found local-only skill: ${agentIdn}/${flowIdn}/${skillIdn}`);
+                            }
+                        }
+                    }
+                    catch (error) {
+                        // Ignore errors reading flow directory
+                    }
+                }
+            }
+            catch (error) {
+                // Ignore errors reading agent directory
+            }
+        }
+    }
+    return { agentCount, flowCount, skillCount, entities: localEntities };
+}
 /**
  * Push changed files to NEWO platform
  */
-export async function pushChanged(client, customer, verbose = false) {
+export async function pushChanged(client, customer, verbose = false, shouldPublish = true) {
     await ensureState(customer.idn);
     if (!(await fs.pathExists(mapPath(customer.idn)))) {
         console.log(`No map for customer ${customer.idn}. Run \`newo pull --customer ${customer.idn}\` first.`);
@@ -33,6 +145,184 @@ export async function pushChanged(client, customer, verbose = false) {
         : isLegacyProjectMap(idMapData)
             ? { '': idMapData }
             : (() => { throw new Error('Invalid project map format'); })();
+    // First, handle any local-only entities (created locally but not yet pushed)
+    const localScan = await scanForLocalOnlyEntities(customer, projects, verbose);
+    const totalLocalEntities = localScan.agentCount + localScan.flowCount + localScan.skillCount;
+    if (totalLocalEntities > 0) {
+        console.log(`📤 Found ${localScan.agentCount} new agent(s), ${localScan.flowCount} new flow(s), ${localScan.skillCount} new skill(s) to create...`);
+        // Process in order: agents first, then flows, then skills
+        const sortedEntities = localScan.entities.sort((a, b) => {
+            const typeOrder = { 'agent': 0, 'flow': 1, 'skill': 2 };
+            return typeOrder[a.type] - typeOrder[b.type];
+        });
+        for (const entity of sortedEntities) {
+            if (entity.type === 'agent') {
+                try {
+                    // Read agent metadata
+                    const metadataContent = await fs.readFile(entity.path, 'utf8');
+                    const metadata = yaml.load(metadataContent);
+                    if (verbose)
+                        console.log(`📤 Creating agent: ${entity.idn}`);
+                    // Get project ID from the project map
+                    const projectData = projects[entity.projectIdn];
+                    if (!projectData?.projectId) {
+                        console.error(`❌ Project ID not found for project: ${entity.projectIdn}`);
+                        continue;
+                    }
+                    // Create agent on NEWO platform using project-specific v2 endpoint
+                    const createAgentRequest = {
+                        idn: metadata.idn,
+                        title: metadata.title || metadata.idn,
+                        description: metadata.description || null,
+                        persona_id: metadata.persona_id || null
+                    };
+                    const createResponse = await createAgent(client, projectData.projectId, createAgentRequest);
+                    console.log(`✅ Agent created: ${entity.idn} (ID: ${createResponse.id})`);
+                    pushed++;
+                    metadataChanged = true;
+                    // Update the metadata with the new ID
+                    metadata.id = createResponse.id;
+                    metadata.updated_at = new Date().toISOString();
+                    const updatedMetadataYaml = yaml.dump(metadata, { indent: 2, quotingType: '"', forceQuotes: false });
+                    await fs.writeFile(entity.path, updatedMetadataYaml);
+                    // Update the project map to include the new agent
+                    if (!projectData.agents[entity.idn]) {
+                        projectData.agents[entity.idn] = {
+                            id: createResponse.id,
+                            flows: {}
+                        };
+                    }
+                }
+                catch (error) {
+                    console.error(`❌ Failed to create agent ${entity.idn}:`, error.response?.data?.message || error.message);
+                }
+            }
+            else if (entity.type === 'flow') {
+                try {
+                    // Read flow metadata
+                    const metadataContent = await fs.readFile(entity.path, 'utf8');
+                    const metadata = yaml.load(metadataContent);
+                    if (verbose)
+                        console.log(`📤 Creating flow: ${entity.agentIdn}/${entity.idn}`);
+                    // Get agent ID from the project map
+                    const projectData = projects[entity.projectIdn];
+                    if (!entity.agentIdn) {
+                        console.error(`❌ Agent IDN missing for flow: ${entity.idn}`);
+                        continue;
+                    }
+                    const agentData = projectData?.agents[entity.agentIdn];
+                    if (!agentData?.id) {
+                        console.error(`❌ Agent ID not found for agent: ${entity.agentIdn}`);
+                        continue;
+                    }
+                    // Create flow on NEWO platform
+                    const createFlowRequest = {
+                        idn: metadata.idn,
+                        title: metadata.title || metadata.idn
+                    };
+                    const createResponse = await createFlow(client, agentData.id, createFlowRequest);
+                    console.log(`✅ Flow created: ${entity.idn} (ID: ${createResponse.id})`);
+                    pushed++;
+                    metadataChanged = true;
+                    // Handle the special case where NEWO flow API returns empty response
+                    if (createResponse.id === 'pending-sync') {
+                        console.log(`✅ Flow created: ${entity.idn} (ID will be synced on next pull)`);
+                        // Mark flow as created but pending ID sync
+                        metadata.id = ''; // Keep empty until sync
+                        metadata.updated_at = new Date().toISOString();
+                        const updatedMetadataYaml = yaml.dump(metadata, { indent: 2, quotingType: '"', forceQuotes: false });
+                        await fs.writeFile(entity.path, updatedMetadataYaml);
+                        // Update the project map with empty ID (will be filled by pull)
+                        if (!agentData.flows[entity.idn]) {
+                            agentData.flows[entity.idn] = {
+                                id: '', // Empty until synced
+                                skills: {}
+                            };
+                        }
+                    }
+                    else {
+                        // Normal case with ID returned
+                        metadata.id = createResponse.id;
+                        metadata.updated_at = new Date().toISOString();
+                        const updatedMetadataYaml = yaml.dump(metadata, { indent: 2, quotingType: '"', forceQuotes: false });
+                        await fs.writeFile(entity.path, updatedMetadataYaml);
+                        // Update the project map to include the new flow
+                        if (!agentData.flows[entity.idn]) {
+                            agentData.flows[entity.idn] = {
+                                id: createResponse.id,
+                                skills: {}
+                            };
+                        }
+                    }
+                }
+                catch (error) {
+                    console.error(`❌ Failed to create flow ${entity.idn}:`, error.response?.data?.message || error.message);
+                }
+            }
+            else if (entity.type === 'skill') {
+                try {
+                    // Read skill metadata
+                    const metadataContent = await fs.readFile(entity.path, 'utf8');
+                    const metadata = yaml.load(metadataContent);
+                    if (verbose)
+                        console.log(`📤 Creating skill: ${entity.agentIdn}/${entity.flowIdn}/${entity.idn}`);
+                    // Get flow ID from the project map
+                    const projectData = projects[entity.projectIdn];
+                    if (!entity.agentIdn || !entity.flowIdn) {
+                        console.error(`❌ Agent IDN or Flow IDN missing for skill: ${entity.idn}`);
+                        continue;
+                    }
+                    const agentData = projectData?.agents[entity.agentIdn];
+                    const flowData = agentData?.flows[entity.flowIdn];
+                    if (!flowData?.id) {
+                        console.error(`❌ Flow ID not found for flow: ${entity.flowIdn}`);
+                        continue;
+                    }
+                    // Read the skill script content
+                    const skillFolderBase = entity.path.replace('/metadata.yaml', '');
+                    const scriptExtension = getExtensionForRunner(metadata.runner_type);
+                    const scriptPath = `${skillFolderBase}/${entity.idn}.${scriptExtension}`;
+                    let scriptContent = '';
+                    if (await fs.pathExists(scriptPath)) {
+                        scriptContent = await fs.readFile(scriptPath, 'utf8');
+                    }
+                    // Create skill on NEWO platform
+                    const createSkillRequest = {
+                        idn: metadata.idn,
+                        title: metadata.title || metadata.idn,
+                        prompt_script: scriptContent,
+                        runner_type: metadata.runner_type,
+                        model: metadata.model,
+                        path: "", // Empty path as shown in curl example
+                        parameters: metadata.parameters || []
+                    };
+                    const createResponse = await createSkill(client, flowData.id, createSkillRequest);
+                    console.log(`✅ Skill created: ${entity.idn} (ID: ${createResponse.id})`);
+                    pushed++;
+                    metadataChanged = true;
+                    // Update the metadata with the new ID
+                    metadata.id = createResponse.id;
+                    metadata.updated_at = new Date().toISOString();
+                    const updatedMetadataYaml = yaml.dump(metadata, { indent: 2, quotingType: '"', forceQuotes: false });
+                    await fs.writeFile(entity.path, updatedMetadataYaml);
+                    // Update the project map to include the new skill
+                    if (!flowData.skills[entity.idn]) {
+                        flowData.skills[entity.idn] = {
+                            id: createResponse.id,
+                            idn: metadata.idn,
+                            title: metadata.title || metadata.idn,
+                            runner_type: metadata.runner_type,
+                            model: metadata.model,
+                            parameters: metadata.parameters || []
+                        };
+                    }
+                }
+                catch (error) {
+                    console.error(`❌ Failed to create skill ${entity.idn}:`, error.response?.data?.message || error.message);
+                }
+            }
+        }
+    }
     for (const [projectIdn, projectData] of Object.entries(projects)) {
         if (verbose && projectIdn)
             console.log(`📁 Checking project: ${projectIdn}`);
@@ -164,8 +454,86 @@ export async function pushChanged(client, customer, verbose = false) {
         const flowsYamlFilePath = flowsYamlPath(customer.idn);
         newHashes[flowsYamlFilePath] = sha256(flowsYamlContent);
     }
+    // Save updated project map if metadata changed (new agents added)
+    if (metadataChanged) {
+        const updatedMapData = isProjectMap(idMapData)
+            ? { projects }
+            : projects['']; // Legacy format
+        if (verbose)
+            console.log(`💾 Saving updated project map...`);
+        await fs.writeJson(mapPath(customer.idn), updatedMapData, { spaces: 2 });
+    }
     // Save updated hashes
     await saveHashes(newHashes, customer.idn);
     console.log(pushed ? `${pushed} file(s) pushed.` : 'No changes to push.');
+    // Publish flows if requested (default behavior)
+    if (shouldPublish && pushed > 0) {
+        if (verbose)
+            console.log('\n🚀 Publishing flows...');
+        let publishedFlows = 0;
+        let failedFlows = 0;
+        const publishErrors = [];
+        for (const [, projectData] of Object.entries(projects)) {
+            for (const [, agentObj] of Object.entries(projectData.agents)) {
+                for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                    if (flowObj.id) {
+                        try {
+                            const publishData = {
+                                version: "1.0",
+                                description: "Published via NEWO CLI",
+                                type: "public"
+                            };
+                            await publishFlow(client, flowObj.id, publishData);
+                            if (verbose)
+                                console.log(`📤 Published flow: ${flowIdn} (${flowObj.id})`);
+                            publishedFlows++;
+                        }
+                        catch (error) {
+                            failedFlows++;
+                            // Extract detailed error information from API response
+                            const errorMessage = error.response?.data?.message || error.message || 'Unknown error';
+                            const errorDetails = error.response?.data?.reasons || error.response?.data?.errors || error.response?.data?.detail;
+                            publishErrors.push({
+                                flowIdn,
+                                error: errorMessage,
+                                details: errorDetails
+                            });
+                            // Always show publish errors (not just in verbose mode)
+                            console.error(`❌ Failed to publish flow '${flowIdn}': ${errorMessage}`);
+                            if (errorDetails) {
+                                if (Array.isArray(errorDetails)) {
+                                    console.error(`   Reasons:`);
+                                    errorDetails.forEach((reason) => {
+                                        console.error(`   • ${reason}`);
+                                    });
+                                }
+                                else if (typeof errorDetails === 'object') {
+                                    console.error(`   Details: ${JSON.stringify(errorDetails, null, 2)}`);
+                                }
+                                else {
+                                    console.error(`   Details: ${errorDetails}`);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        // Summary message
+        if (publishedFlows > 0 || failedFlows > 0) {
+            console.log(`\n🚀 Publish summary: ${publishedFlows} succeeded, ${failedFlows} failed.`);
+            if (failedFlows > 0) {
+                console.log(`\n⚠️  ${failedFlows} flow(s) failed to publish due to validation errors.`);
+                console.log(`   Fix the errors above and run 'npm run push' again.`);
+            }
+        }
+        else if (verbose) {
+            console.log('\n💡 No flows to publish.');
+        }
+    }
+    // If we created flows, recommend a pull to sync flow IDs
+    if (localScan.flowCount > 0) {
+        console.log('\n💡 Tip: Run "newo pull" to sync flow IDs and enable skill creation.');
+    }
 }
 //# sourceMappingURL=push.js.map

package/dist/sync/skill-files.js

@@ -107,62 +107,35 @@ export async function askForOverwrite(skillIdn, existingContent, newContent, fil
     const greenBg = '\x1b[102m\x1b[30m'; // Light green background, black text (like GitHub)
     const gray = '\x1b[90m';
     const reset = '\x1b[0m';
-    // Show a GitHub-style colored diff with line numbers and context
-    const localLines = existingContent.trim().split('\n');
-    const remoteLines = newContent.trim().split('\n');
-    // Find the first differing section and show it with context
-    let diffStart = -1;
-    let diffEnd = -1;
-    // Find first difference
-    for (let i = 0; i < Math.max(localLines.length, remoteLines.length); i++) {
-        if (localLines[i] !== remoteLines[i]) {
-            diffStart = i;
-            break;
-        }
-    }
-    if (diffStart === -1) {
-        // No differences found (shouldn't happen, but handle gracefully)
+    // Generate proper diff using LCS algorithm
+    const { generateDiff, filterDiffWithContext } = await import('./diff-utils.js');
+    const fullDiff = generateDiff(existingContent, newContent);
+    const contextDiff = filterDiffWithContext(fullDiff, 2);
+    if (contextDiff.length === 0) {
         console.log(`${gray}   No differences found${reset}`);
         return 'no';
     }
-    // Find end of difference section
-    diffEnd = diffStart;
-    while (diffEnd < Math.max(localLines.length, remoteLines.length) &&
-        (localLines[diffEnd] !== remoteLines[diffEnd] ||
-            localLines[diffEnd] === undefined ||
-            remoteLines[diffEnd] === undefined)) {
-        diffEnd++;
-    }
-    // Show context before (2 lines)
-    const contextStart = Math.max(0, diffStart - 2);
-    for (let i = contextStart; i < diffStart; i++) {
-        if (localLines[i] !== undefined) {
-            console.log(`    ${String(i + 1).padStart(3)}      ${localLines[i]}`);
+    // Display the diff with proper GitHub-style formatting
+    for (const line of contextDiff) {
+        if (line.type === 'context') {
+            // Show context lines in gray
+            const lineNum = line.localLineNum !== -1 ? line.localLineNum : line.remoteLineNum;
+            console.log(`    ${String(lineNum).padStart(3)}      ${line.content}`);
         }
-    }
-    // Show the differences
-    const maxDiffLine = Math.min(diffEnd, Math.max(localLines.length, remoteLines.length));
-    for (let i = diffStart; i < maxDiffLine; i++) {
-        const localLine = localLines[i];
-        const remoteLine = remoteLines[i];
-        if (localLine !== undefined && (remoteLine === undefined || localLine !== remoteLine)) {
-            console.log(`${redBg} -  ${String(i + 1).padStart(3)}      ${localLine} ${reset}`);
-        }
-        if (remoteLine !== undefined && (localLine === undefined || localLine !== remoteLine)) {
-            console.log(`${greenBg} +  ${String(i + 1).padStart(3)}      ${remoteLine} ${reset}`);
+        else if (line.type === 'remove') {
+            // Show local content being removed (red background)
+            console.log(`${redBg} -  ${String(line.localLineNum).padStart(3)}      ${line.content} ${reset}`);
         }
-    }
-    // Show context after (2 lines)
-    const contextEnd = Math.min(localLines.length, diffEnd + 2);
-    for (let i = diffEnd; i < contextEnd; i++) {
-        if (localLines[i] !== undefined) {
-            console.log(`    ${String(i + 1).padStart(3)}      ${localLines[i]}`);
+        else if (line.type === 'add') {
+            // Show remote content being added (green background)
+            console.log(`${greenBg} +  ${String(line.remoteLineNum).padStart(3)}      ${line.content} ${reset}`);
         }
     }
-    // Show if there are more differences
-    const totalDiffs = Math.abs(localLines.length - remoteLines.length) + (diffEnd - diffStart);
-    if (totalDiffs > (diffEnd - diffStart)) {
-        console.log(`${gray}... (${totalDiffs - (diffEnd - diffStart)} more differences)${reset}`);
+    // Show if there are more changes beyond what we're displaying
+    const totalChanges = fullDiff.filter(line => line.type !== 'context').length;
+    const displayedChanges = contextDiff.filter(line => line.type !== 'context').length;
+    if (totalChanges > displayedChanges) {
+        console.log(`${gray}... (${totalChanges - displayedChanges} more changes)${reset}`);
     }
     const answer = await new Promise((resolve) => {
         rl.question(`\nReplace local with remote? (y)es/(n)o/(a)ll/(q)uit: `, resolve);