newo 2.0.5 → 3.0.0

package/src/sync/push.ts

@@ -1,15 +1,18 @@
 /**
  * Push operations for changed files
  */
-import { updateSkill } from '../api.js';
+import { updateSkill, createAgent, createFlow, createSkill, publishFlow } from '../api.js';
 import {
   ensureState,
   mapPath,
-  skillMetadataPath
+  skillMetadataPath,
+  projectDir,
+  agentMetadataPath
 } from '../fsutil.js';
 import {
   validateSkillFolder,
-  getSingleSkillFile
+  getSingleSkillFile,
+  getExtensionForRunner
 } from './skill-files.js';
 import fs from 'fs-extra';
 import { sha256, loadHashes, saveHashes } from '../hash.js';
@@ -22,13 +25,135 @@ import type {
   ProjectData,
   ProjectMap,
   CustomerConfig,
-  SkillMetadata
+  SkillMetadata,
+  AgentMetadata,
+  FlowMetadata,
+  CreateAgentRequest,
+  CreateFlowRequest,
+  CreateSkillRequest,
+  PublishFlowRequest
 } from '../types.js';
 
+/**
+ * Scan filesystem for local-only entities not in the project map yet
+ */
+async function scanForLocalOnlyEntities(customer: CustomerConfig, projects: Record<string, ProjectData>, verbose: boolean = false): Promise<{ agentCount: number; flowCount: number; skillCount: number; entities: Array<{ type: 'agent' | 'flow' | 'skill'; path: string; idn: string; projectIdn: string; agentIdn?: string; flowIdn?: string }> }> {
+  const localEntities: Array<{ type: 'agent' | 'flow' | 'skill'; path: string; idn: string; projectIdn: string; agentIdn?: string; flowIdn?: string }> = [];
+  let agentCount = 0;
+  let flowCount = 0;
+  let skillCount = 0;
+
+  // Scan each project directory
+  for (const [projectIdn] of Object.entries(projects)) {
+    const projDir = projectDir(customer.idn, projectIdn);
+    if (!(await fs.pathExists(projDir))) continue;
+
+    if (verbose) console.log(`🔍 Scanning project directory: ${projDir}`);
+
+    // Get all subdirectories in the project (these should be agents)
+    const agentDirs = await fs.readdir(projDir);
+
+    for (const agentIdn of agentDirs) {
+      const agentPath = `${projDir}/${agentIdn}`;
+      const agentStat = await fs.stat(agentPath);
+
+      // Skip files, only process directories
+      if (!agentStat.isDirectory()) continue;
+
+      // Skip if it's not really an agent directory (no metadata.yaml)
+      const agentMetaPath = agentMetadataPath(customer.idn, projectIdn, agentIdn);
+      if (!(await fs.pathExists(agentMetaPath))) continue;
+
+      // Check if this agent is already in the project map
+      const projectData = projects[projectIdn];
+      if (!projectData?.agents[agentIdn]) {
+        // This is a local-only agent!
+        localEntities.push({
+          type: 'agent',
+          path: agentMetaPath,
+          idn: agentIdn,
+          projectIdn
+        });
+        agentCount++;
+        if (verbose) console.log(`  🆕 Found local-only agent: ${agentIdn}`);
+      }
+
+      // Now scan for flows within this agent (regardless of whether agent is local-only or not)
+      try {
+        const flowDirs = await fs.readdir(agentPath);
+        for (const flowIdn of flowDirs) {
+          const flowPath = `${agentPath}/${flowIdn}`;
+          const flowStat = await fs.stat(flowPath);
+
+          // Skip files, only process directories
+          if (!flowStat.isDirectory()) continue;
+
+          // Skip if it's not really a flow directory (no metadata.yaml)
+          const flowMetaPath = `${flowPath}/metadata.yaml`;
+          if (!(await fs.pathExists(flowMetaPath))) continue;
+
+          // Check if this flow exists in the project map
+          const agentData = projectData?.agents[agentIdn];
+          if (!agentData?.flows[flowIdn]) {
+            // This is a local-only flow!
+            localEntities.push({
+              type: 'flow',
+              path: flowMetaPath,
+              idn: flowIdn,
+              projectIdn,
+              agentIdn
+            });
+            flowCount++;
+            if (verbose) console.log(`    🆕 Found local-only flow: ${agentIdn}/${flowIdn}`);
+          }
+
+          // Now scan for skills within this flow (regardless of whether flow is local-only or not)
+          try {
+            const skillDirs = await fs.readdir(flowPath);
+            for (const skillIdn of skillDirs) {
+              const skillPath = `${flowPath}/${skillIdn}`;
+              const skillStat = await fs.stat(skillPath);
+
+              // Skip files, only process directories
+              if (!skillStat.isDirectory()) continue;
+
+              // Skip if it's not really a skill directory (no metadata.yaml)
+              const skillMetaPath = `${skillPath}/metadata.yaml`;
+              if (!(await fs.pathExists(skillMetaPath))) continue;
+
+              // Check if this skill exists in the project map
+              const flowData = agentData?.flows[flowIdn];
+              if (!flowData?.skills[skillIdn]) {
+                // This is a local-only skill!
+                localEntities.push({
+                  type: 'skill',
+                  path: skillMetaPath,
+                  idn: skillIdn,
+                  projectIdn,
+                  agentIdn,
+                  flowIdn
+                });
+                skillCount++;
+                if (verbose) console.log(`      🆕 Found local-only skill: ${agentIdn}/${flowIdn}/${skillIdn}`);
+              }
+            }
+          } catch (error) {
+            // Ignore errors reading flow directory
+          }
+        }
+      } catch (error) {
+        // Ignore errors reading agent directory
+      }
+    }
+  }
+
+  return { agentCount, flowCount, skillCount, entities: localEntities };
+}
+
 /**
  * Push changed files to NEWO platform
  */
-export async function pushChanged(client: AxiosInstance, customer: CustomerConfig, verbose: boolean = false): Promise<void> {
+export async function pushChanged(client: AxiosInstance, customer: CustomerConfig, verbose: boolean = false, shouldPublish: boolean = true): Promise<void> {
   await ensureState(customer.idn);
   if (!(await fs.pathExists(mapPath(customer.idn)))) {
     console.log(`No map for customer ${customer.idn}. Run \`newo pull --customer ${customer.idn}\` first.`);
@@ -50,6 +175,205 @@ export async function pushChanged(client: AxiosInstance, customer: CustomerConfi
     ? { '': idMapData as ProjectData }
     : (() => { throw new Error('Invalid project map format'); })();
 
+  // First, handle any local-only entities (created locally but not yet pushed)
+  const localScan = await scanForLocalOnlyEntities(customer, projects, verbose);
+  const totalLocalEntities = localScan.agentCount + localScan.flowCount + localScan.skillCount;
+
+  if (totalLocalEntities > 0) {
+    console.log(`📤 Found ${localScan.agentCount} new agent(s), ${localScan.flowCount} new flow(s), ${localScan.skillCount} new skill(s) to create...`);
+
+    // Process in order: agents first, then flows, then skills
+    const sortedEntities = localScan.entities.sort((a, b) => {
+      const typeOrder = { 'agent': 0, 'flow': 1, 'skill': 2 };
+      return typeOrder[a.type] - typeOrder[b.type];
+    });
+
+    for (const entity of sortedEntities) {
+      if (entity.type === 'agent') {
+        try {
+          // Read agent metadata
+          const metadataContent = await fs.readFile(entity.path, 'utf8');
+          const metadata = yaml.load(metadataContent) as AgentMetadata;
+
+          if (verbose) console.log(`📤 Creating agent: ${entity.idn}`);
+
+          // Get project ID from the project map
+          const projectData = projects[entity.projectIdn];
+          if (!projectData?.projectId) {
+            console.error(`❌ Project ID not found for project: ${entity.projectIdn}`);
+            continue;
+          }
+
+          // Create agent on NEWO platform using project-specific v2 endpoint
+          const createAgentRequest: CreateAgentRequest = {
+            idn: metadata.idn,
+            title: metadata.title || metadata.idn,
+            description: metadata.description || null,
+            persona_id: metadata.persona_id || null
+          };
+
+          const createResponse = await createAgent(client, projectData.projectId, createAgentRequest);
+          console.log(`✅ Agent created: ${entity.idn} (ID: ${createResponse.id})`);
+          pushed++;
+          metadataChanged = true;
+
+          // Update the metadata with the new ID
+          metadata.id = createResponse.id;
+          metadata.updated_at = new Date().toISOString();
+          const updatedMetadataYaml = yaml.dump(metadata, { indent: 2, quotingType: '"', forceQuotes: false });
+          await fs.writeFile(entity.path, updatedMetadataYaml);
+
+          // Update the project map to include the new agent
+          if (!projectData.agents[entity.idn]) {
+            projectData.agents[entity.idn] = {
+              id: createResponse.id,
+              flows: {}
+            };
+          }
+
+        } catch (error: any) {
+          console.error(`❌ Failed to create agent ${entity.idn}:`, error.response?.data?.message || error.message);
+        }
+
+      } else if (entity.type === 'flow') {
+        try {
+          // Read flow metadata
+          const metadataContent = await fs.readFile(entity.path, 'utf8');
+          const metadata = yaml.load(metadataContent) as FlowMetadata;
+
+          if (verbose) console.log(`📤 Creating flow: ${entity.agentIdn}/${entity.idn}`);
+
+          // Get agent ID from the project map
+          const projectData = projects[entity.projectIdn];
+          if (!entity.agentIdn) {
+            console.error(`❌ Agent IDN missing for flow: ${entity.idn}`);
+            continue;
+          }
+          const agentData = projectData?.agents[entity.agentIdn];
+          if (!agentData?.id) {
+            console.error(`❌ Agent ID not found for agent: ${entity.agentIdn}`);
+            continue;
+          }
+
+          // Create flow on NEWO platform
+          const createFlowRequest: CreateFlowRequest = {
+            idn: metadata.idn,
+            title: metadata.title || metadata.idn
+          };
+
+          const createResponse = await createFlow(client, agentData.id, createFlowRequest);
+          console.log(`✅ Flow created: ${entity.idn} (ID: ${createResponse.id})`);
+          pushed++;
+          metadataChanged = true;
+
+          // Handle the special case where NEWO flow API returns empty response
+          if (createResponse.id === 'pending-sync') {
+            console.log(`✅ Flow created: ${entity.idn} (ID will be synced on next pull)`);
+            // Mark flow as created but pending ID sync
+            metadata.id = '';  // Keep empty until sync
+            metadata.updated_at = new Date().toISOString();
+            const updatedMetadataYaml = yaml.dump(metadata, { indent: 2, quotingType: '"', forceQuotes: false });
+            await fs.writeFile(entity.path, updatedMetadataYaml);
+
+            // Update the project map with empty ID (will be filled by pull)
+            if (!agentData.flows[entity.idn]) {
+              agentData.flows[entity.idn] = {
+                id: '',  // Empty until synced
+                skills: {}
+              };
+            }
+          } else {
+            // Normal case with ID returned
+            metadata.id = createResponse.id;
+            metadata.updated_at = new Date().toISOString();
+            const updatedMetadataYaml = yaml.dump(metadata, { indent: 2, quotingType: '"', forceQuotes: false });
+            await fs.writeFile(entity.path, updatedMetadataYaml);
+
+            // Update the project map to include the new flow
+            if (!agentData.flows[entity.idn]) {
+              agentData.flows[entity.idn] = {
+                id: createResponse.id,
+                skills: {}
+              };
+            }
+          }
+
+        } catch (error: any) {
+          console.error(`❌ Failed to create flow ${entity.idn}:`, error.response?.data?.message || error.message);
+        }
+
+      } else if (entity.type === 'skill') {
+        try {
+          // Read skill metadata
+          const metadataContent = await fs.readFile(entity.path, 'utf8');
+          const metadata = yaml.load(metadataContent) as SkillMetadata;
+
+          if (verbose) console.log(`📤 Creating skill: ${entity.agentIdn}/${entity.flowIdn}/${entity.idn}`);
+
+          // Get flow ID from the project map
+          const projectData = projects[entity.projectIdn];
+          if (!entity.agentIdn || !entity.flowIdn) {
+            console.error(`❌ Agent IDN or Flow IDN missing for skill: ${entity.idn}`);
+            continue;
+          }
+          const agentData = projectData?.agents[entity.agentIdn];
+          const flowData = agentData?.flows[entity.flowIdn];
+          if (!flowData?.id) {
+            console.error(`❌ Flow ID not found for flow: ${entity.flowIdn}`);
+            continue;
+          }
+
+          // Read the skill script content
+          const skillFolderBase = entity.path.replace('/metadata.yaml', '');
+          const scriptExtension = getExtensionForRunner(metadata.runner_type);
+          const scriptPath = `${skillFolderBase}/${entity.idn}.${scriptExtension}`;
+
+          let scriptContent = '';
+          if (await fs.pathExists(scriptPath)) {
+            scriptContent = await fs.readFile(scriptPath, 'utf8');
+          }
+
+          // Create skill on NEWO platform
+          const createSkillRequest: CreateSkillRequest = {
+            idn: metadata.idn,
+            title: metadata.title || metadata.idn,
+            prompt_script: scriptContent,
+            runner_type: metadata.runner_type,
+            model: metadata.model,
+            path: "",  // Empty path as shown in curl example
+            parameters: metadata.parameters || []
+          };
+
+          const createResponse = await createSkill(client, flowData.id, createSkillRequest);
+          console.log(`✅ Skill created: ${entity.idn} (ID: ${createResponse.id})`);
+          pushed++;
+          metadataChanged = true;
+
+          // Update the metadata with the new ID
+          metadata.id = createResponse.id;
+          metadata.updated_at = new Date().toISOString();
+          const updatedMetadataYaml = yaml.dump(metadata, { indent: 2, quotingType: '"', forceQuotes: false });
+          await fs.writeFile(entity.path, updatedMetadataYaml);
+
+          // Update the project map to include the new skill
+          if (!flowData.skills[entity.idn]) {
+            flowData.skills[entity.idn] = {
+              id: createResponse.id,
+              idn: metadata.idn,
+              title: metadata.title || metadata.idn,
+              runner_type: metadata.runner_type,
+              model: metadata.model,
+              parameters: metadata.parameters || []
+            };
+          }
+
+        } catch (error: any) {
+          console.error(`❌ Failed to create skill ${entity.idn}:`, error.response?.data?.message || error.message);
+        }
+      }
+    }
+  }
+
   for (const [projectIdn, projectData] of Object.entries(projects)) {
     if (verbose && projectIdn) console.log(`📁 Checking project: ${projectIdn}`);
 
@@ -193,8 +517,92 @@ export async function pushChanged(client: AxiosInstance, customer: CustomerConfi
     newHashes[flowsYamlFilePath] = sha256(flowsYamlContent);
   }
 
+  // Save updated project map if metadata changed (new agents added)
+  if (metadataChanged) {
+    const updatedMapData = isProjectMap(idMapData)
+      ? { projects } as ProjectMap
+      : projects[''] as ProjectData; // Legacy format
+
+    if (verbose) console.log(`💾 Saving updated project map...`);
+    await fs.writeJson(mapPath(customer.idn), updatedMapData, { spaces: 2 });
+  }
+
   // Save updated hashes
   await saveHashes(newHashes, customer.idn);
 
   console.log(pushed ? `${pushed} file(s) pushed.` : 'No changes to push.');
+
+  // Publish flows if requested (default behavior)
+  if (shouldPublish && pushed > 0) {
+    if (verbose) console.log('\n🚀 Publishing flows...');
+
+    let publishedFlows = 0;
+    let failedFlows = 0;
+    const publishErrors: Array<{ flowIdn: string; error: string; details?: any }> = [];
+
+    for (const [, projectData] of Object.entries(projects)) {
+      for (const [, agentObj] of Object.entries(projectData.agents)) {
+        for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+          if (flowObj.id) {
+            try {
+              const publishData: PublishFlowRequest = {
+                version: "1.0",
+                description: "Published via NEWO CLI",
+                type: "public"
+              };
+
+              await publishFlow(client, flowObj.id, publishData);
+              if (verbose) console.log(`📤 Published flow: ${flowIdn} (${flowObj.id})`);
+              publishedFlows++;
+            } catch (error: any) {
+              failedFlows++;
+
+              // Extract detailed error information from API response
+              const errorMessage = error.response?.data?.message || error.message || 'Unknown error';
+              const errorDetails = error.response?.data?.reasons || error.response?.data?.errors || error.response?.data?.detail;
+
+              publishErrors.push({
+                flowIdn,
+                error: errorMessage,
+                details: errorDetails
+              });
+
+              // Always show publish errors (not just in verbose mode)
+              console.error(`❌ Failed to publish flow '${flowIdn}': ${errorMessage}`);
+
+              if (errorDetails) {
+                if (Array.isArray(errorDetails)) {
+                  console.error(`   Reasons:`);
+                  errorDetails.forEach((reason: string) => {
+                    console.error(`   • ${reason}`);
+                  });
+                } else if (typeof errorDetails === 'object') {
+                  console.error(`   Details: ${JSON.stringify(errorDetails, null, 2)}`);
+                } else {
+                  console.error(`   Details: ${errorDetails}`);
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    // Summary message
+    if (publishedFlows > 0 || failedFlows > 0) {
+      console.log(`\n🚀 Publish summary: ${publishedFlows} succeeded, ${failedFlows} failed.`);
+
+      if (failedFlows > 0) {
+        console.log(`\n⚠️  ${failedFlows} flow(s) failed to publish due to validation errors.`);
+        console.log(`   Fix the errors above and run 'npm run push' again.`);
+      }
+    } else if (verbose) {
+      console.log('\n💡 No flows to publish.');
+    }
+  }
+
+  // If we created flows, recommend a pull to sync flow IDs
+  if (localScan.flowCount > 0) {
+    console.log('\n💡 Tip: Run "newo pull" to sync flow IDs and enable skill creation.');
+  }
 }

package/src/sync/skill-files.ts

@@ -162,71 +162,36 @@ export async function askForOverwrite(skillIdn: string, existingContent: string,
   const gray = '\x1b[90m';
   const reset = '\x1b[0m';
 
-  // Show a GitHub-style colored diff with line numbers and context
-  const localLines = existingContent.trim().split('\n');
-  const remoteLines = newContent.trim().split('\n');
-
-  // Find the first differing section and show it with context
-  let diffStart = -1;
-  let diffEnd = -1;
-
-  // Find first difference
-  for (let i = 0; i < Math.max(localLines.length, remoteLines.length); i++) {
-    if (localLines[i] !== remoteLines[i]) {
-      diffStart = i;
-      break;
-    }
-  }
+  // Generate proper diff using LCS algorithm
+  const { generateDiff, filterDiffWithContext } = await import('./diff-utils.js');
+  const fullDiff = generateDiff(existingContent, newContent);
+  const contextDiff = filterDiffWithContext(fullDiff, 2);
 
-  if (diffStart === -1) {
-    // No differences found (shouldn't happen, but handle gracefully)
+  if (contextDiff.length === 0) {
     console.log(`${gray}   No differences found${reset}`);
     return 'no';
   }
 
-  // Find end of difference section
-  diffEnd = diffStart;
-  while (diffEnd < Math.max(localLines.length, remoteLines.length) &&
-         (localLines[diffEnd] !== remoteLines[diffEnd] ||
-          localLines[diffEnd] === undefined ||
-          remoteLines[diffEnd] === undefined)) {
-    diffEnd++;
-  }
-
-  // Show context before (2 lines)
-  const contextStart = Math.max(0, diffStart - 2);
-  for (let i = contextStart; i < diffStart; i++) {
-    if (localLines[i] !== undefined) {
-      console.log(`    ${String(i + 1).padStart(3)}      ${localLines[i]}`);
-    }
-  }
-
-  // Show the differences
-  const maxDiffLine = Math.min(diffEnd, Math.max(localLines.length, remoteLines.length));
-  for (let i = diffStart; i < maxDiffLine; i++) {
-    const localLine = localLines[i];
-    const remoteLine = remoteLines[i];
-
-    if (localLine !== undefined && (remoteLine === undefined || localLine !== remoteLine)) {
-      console.log(`${redBg} -  ${String(i + 1).padStart(3)}      ${localLine} ${reset}`);
-    }
-    if (remoteLine !== undefined && (localLine === undefined || localLine !== remoteLine)) {
-      console.log(`${greenBg} +  ${String(i + 1).padStart(3)}      ${remoteLine} ${reset}`);
-    }
-  }
-
-  // Show context after (2 lines)
-  const contextEnd = Math.min(localLines.length, diffEnd + 2);
-  for (let i = diffEnd; i < contextEnd; i++) {
-    if (localLines[i] !== undefined) {
-      console.log(`    ${String(i + 1).padStart(3)}      ${localLines[i]}`);
+  // Display the diff with proper GitHub-style formatting
+  for (const line of contextDiff) {
+    if (line.type === 'context') {
+      // Show context lines in gray
+      const lineNum = line.localLineNum !== -1 ? line.localLineNum : line.remoteLineNum;
+      console.log(`    ${String(lineNum).padStart(3)}      ${line.content}`);
+    } else if (line.type === 'remove') {
+      // Show local content being removed (red background)
+      console.log(`${redBg} -  ${String(line.localLineNum).padStart(3)}      ${line.content} ${reset}`);
+    } else if (line.type === 'add') {
+      // Show remote content being added (green background)
+      console.log(`${greenBg} +  ${String(line.remoteLineNum).padStart(3)}      ${line.content} ${reset}`);
     }
   }
 
-  // Show if there are more differences
-  const totalDiffs = Math.abs(localLines.length - remoteLines.length) + (diffEnd - diffStart);
-  if (totalDiffs > (diffEnd - diffStart)) {
-    console.log(`${gray}... (${totalDiffs - (diffEnd - diffStart)} more differences)${reset}`);
+  // Show if there are more changes beyond what we're displaying
+  const totalChanges = fullDiff.filter(line => line.type !== 'context').length;
+  const displayedChanges = contextDiff.filter(line => line.type !== 'context').length;
+  if (totalChanges > displayedChanges) {
+    console.log(`${gray}... (${totalChanges - displayedChanges} more changes)${reset}`);
   }
 
   const answer = await new Promise<string>((resolve) => {