newo 1.3.0 → 1.5.0

package/src/{cli.js → cli.ts}

@@ -5,29 +5,37 @@ import { makeClient, getProjectMeta, importAkbArticle } from './api.js';
 import { pullAll, pushChanged, status } from './sync.js';
 import { parseAkbFile, prepareArticlesForImport } from './akb.js';
 import path from 'path';
+import type { NewoEnvironment, CliArgs, NewoApiError } from './types.js';
 
 dotenv.config();
-const { NEWO_PROJECT_ID } = process.env;
+const { NEWO_PROJECT_ID } = process.env as NewoEnvironment;
 
-async function main() {
-  const args = minimist(process.argv.slice(2));
+async function main(): Promise<void> {
+  const args = minimist(process.argv.slice(2)) as CliArgs;
   const cmd = args._[0];
-  const verbose = args.verbose || args.v;
+  const verbose = Boolean(args.verbose || args.v);
 
   if (!cmd || ['help', '-h', '--help'].includes(cmd)) {
     console.log(`NEWO CLI
 Usage:
-  newo pull                    # download project -> ./project
+  newo pull                    # download all projects -> ./projects/ OR specific project if NEWO_PROJECT_ID set
   newo push                    # upload modified *.guidance/*.jinja back to NEWO
   newo status                  # show modified files
-  newo meta                    # get project metadata (debug)
+  newo meta                    # get project metadata (debug, requires NEWO_PROJECT_ID)
   newo import-akb <file> <persona_id>  # import AKB articles from file
   
 Flags:
   --verbose, -v                # enable detailed logging
   
 Env:
-  NEWO_BASE_URL, NEWO_PROJECT_ID, NEWO_API_KEY, NEWO_REFRESH_URL (optional)
+  NEWO_BASE_URL, NEWO_PROJECT_ID (optional), NEWO_API_KEY, NEWO_REFRESH_URL (optional)
+  
+Notes:
+  - multi-project support: pull downloads all accessible projects or single project based on NEWO_PROJECT_ID
+  - If NEWO_PROJECT_ID is set, pull downloads only that project
+  - If NEWO_PROJECT_ID is not set, pull downloads all projects accessible with your API key
+  - Projects are stored in ./projects/{project-idn}/ folders
+  - Each project folder contains metadata.json and flows.yaml
 `);
     return;
   }
@@ -35,14 +43,16 @@ Env:
   const client = await makeClient(verbose);
 
   if (cmd === 'pull') {
-    if (!NEWO_PROJECT_ID) throw new Error('NEWO_PROJECT_ID is not set in env');
-    await pullAll(client, NEWO_PROJECT_ID, verbose);
+    // If PROJECT_ID is set, pull single project; otherwise pull all projects
+    await pullAll(client, NEWO_PROJECT_ID || null, verbose);
   } else if (cmd === 'push') {
     await pushChanged(client, verbose);
   } else if (cmd === 'status') {
     await status(verbose);
   } else if (cmd === 'meta') {
-    if (!NEWO_PROJECT_ID) throw new Error('NEWO_PROJECT_ID is not set in env');
+    if (!NEWO_PROJECT_ID) {
+      throw new Error('NEWO_PROJECT_ID is not set in env');
+    }
     const meta = await getProjectMeta(client, NEWO_PROJECT_ID);
     console.log(JSON.stringify(meta, null, 2));
   } else if (cmd === 'import-akb') {
@@ -72,13 +82,16 @@ Env:
       
       for (const [index, article] of preparedArticles.entries()) {
         try {
-          if (verbose) console.log(`  [${index + 1}/${preparedArticles.length}] Importing ${article.topic_name}...`);
+          if (verbose) {
+            console.log(`  [${index + 1}/${preparedArticles.length}] Importing ${article.topic_name}...`);
+          }
           await importAkbArticle(client, article);
           successCount++;
           if (!verbose) process.stdout.write('.');
         } catch (error) {
           errorCount++;
-          console.error(`\n❌ Failed to import ${article.topic_name}:`, error?.response?.data || error.message);
+          const errorMessage = (error as NewoApiError)?.response?.data || (error as Error).message;
+          console.error(`\n❌ Failed to import ${article.topic_name}:`, errorMessage);
         }
       }
       
@@ -86,7 +99,7 @@ Env:
       console.log(`✅ Import complete: ${successCount} successful, ${errorCount} failed`);
       
     } catch (error) {
-      console.error('❌ AKB import failed:', error.message);
+      console.error('❌ AKB import failed:', (error as Error).message);
       process.exit(1);
     }
   } else {
@@ -95,7 +108,8 @@ Env:
   }
 }
 
-main().catch((e) => {
-  console.error(e?.response?.data || e);
+main().catch((error: NewoApiError | Error) => {
+  const errorData = 'response' in error ? error?.response?.data : error;
+  console.error(errorData || error);
   process.exit(1);
 });

package/src/fsutil.ts

@@ -0,0 +1,41 @@
+import fs from 'fs-extra';
+import path from 'path';
+import type { RunnerType } from './types.js';
+
+export const ROOT_DIR = path.join(process.cwd(), 'projects');
+export const STATE_DIR = path.join(process.cwd(), '.newo');
+export const MAP_PATH = path.join(STATE_DIR, 'map.json');
+export const HASHES_PATH = path.join(STATE_DIR, 'hashes.json');
+
+export async function ensureState(): Promise<void> {
+  await fs.ensureDir(STATE_DIR);
+  await fs.ensureDir(ROOT_DIR);
+}
+
+export function projectDir(projectIdn: string): string {
+  return path.join(ROOT_DIR, projectIdn);
+}
+
+export function skillPath(
+  projectIdn: string, 
+  agentIdn: string, 
+  flowIdn: string, 
+  skillIdn: string, 
+  runnerType: RunnerType = 'guidance'
+): string {
+  const extension = runnerType === 'nsl' ? '.jinja' : '.guidance';
+  return path.join(ROOT_DIR, projectIdn, agentIdn, flowIdn, `${skillIdn}${extension}`);
+}
+
+export function metadataPath(projectIdn: string): string {
+  return path.join(ROOT_DIR, projectIdn, 'metadata.json');
+}
+
+export async function writeFileAtomic(filepath: string, content: string): Promise<void> {
+  await fs.ensureDir(path.dirname(filepath));
+  await fs.writeFile(filepath, content, 'utf8');
+}
+
+export async function readIfExists(filepath: string): Promise<string | null> {
+  return (await fs.pathExists(filepath)) ? fs.readFile(filepath, 'utf8') : null;
+}

package/src/hash.ts

@@ -0,0 +1,20 @@
+import crypto from 'crypto';
+import fs from 'fs-extra';
+import { ensureState, HASHES_PATH } from './fsutil.js';
+import type { HashStore } from './types.js';
+
+export function sha256(str: string): string {
+  return crypto.createHash('sha256').update(str, 'utf8').digest('hex');
+}
+
+export async function loadHashes(): Promise<HashStore> {
+  await ensureState();
+  if (await fs.pathExists(HASHES_PATH)) {
+    return fs.readJson(HASHES_PATH) as Promise<HashStore>;
+  }
+  return {};
+}
+
+export async function saveHashes(hashes: HashStore): Promise<void> {
+  await fs.writeJson(HASHES_PATH, hashes, { spaces: 2 });
+}

package/src/sync.ts

@@ -0,0 +1,396 @@
+import { 
+  listProjects, 
+  listAgents, 
+  listFlowSkills, 
+  updateSkill, 
+  listFlowEvents, 
+  listFlowStates, 
+  getProjectMeta 
+} from './api.js';
+import { 
+  ensureState, 
+  skillPath, 
+  writeFileAtomic, 
+  readIfExists, 
+  MAP_PATH, 
+  projectDir, 
+  metadataPath 
+} from './fsutil.js';
+import fs from 'fs-extra';
+import { sha256, loadHashes, saveHashes } from './hash.js';
+import yaml from 'js-yaml';
+import path from 'path';
+import type { AxiosInstance } from 'axios';
+import type { 
+  Agent, 
+  ProjectData, 
+  ProjectMap, 
+  LegacyProjectMap,
+  HashStore,
+  FlowsYamlData,
+  FlowsYamlAgent,
+  FlowsYamlFlow,
+  FlowsYamlSkill,
+  FlowsYamlEvent,
+  FlowsYamlState
+} from './types.js';
+
+export async function pullSingleProject(
+  client: AxiosInstance, 
+  projectId: string, 
+  projectIdn: string, 
+  verbose: boolean = false
+): Promise<ProjectData> {
+  if (verbose) console.log(`🔍 Fetching agents for project ${projectId} (${projectIdn})...`);
+  const agents = await listAgents(client, projectId);
+  if (verbose) console.log(`📦 Found ${agents.length} agents`);
+
+  // Get and save project metadata
+  const projectMeta = await getProjectMeta(client, projectId);
+  await writeFileAtomic(metadataPath(projectIdn), JSON.stringify(projectMeta, null, 2));
+  if (verbose) console.log(`✓ Saved metadata for ${projectIdn}`);
+
+  const projectMap: ProjectData = { projectId, projectIdn, agents: {} };
+
+  for (const agent of agents) {
+    const aKey = agent.idn;
+    projectMap.agents[aKey] = { id: agent.id, flows: {} };
+
+    for (const flow of agent.flows ?? []) {
+      projectMap.agents[aKey]!.flows[flow.idn] = { id: flow.id, skills: {} };
+
+      const skills = await listFlowSkills(client, flow.id);
+      for (const skill of skills) {
+        const file = skillPath(projectIdn, agent.idn, flow.idn, skill.idn, skill.runner_type);
+        await writeFileAtomic(file, skill.prompt_script || '');
+        
+        // Store complete skill metadata for push operations
+        projectMap.agents[aKey]!.flows[flow.idn]!.skills[skill.idn] = {
+          id: skill.id,
+          title: skill.title,
+          idn: skill.idn,
+          runner_type: skill.runner_type,
+          model: skill.model,
+          parameters: skill.parameters,
+          path: skill.path || undefined
+        };
+        console.log(`✓ Pulled ${file}`);
+      }
+    }
+  }
+
+  // Generate flows.yaml for this project
+  if (verbose) console.log(`📄 Generating flows.yaml for ${projectIdn}...`);
+  await generateFlowsYaml(client, agents, projectIdn, verbose);
+
+  return projectMap;
+}
+
+export async function pullAll(
+  client: AxiosInstance, 
+  projectId: string | null = null, 
+  verbose: boolean = false
+): Promise<void> {
+  await ensureState();
+  
+  if (projectId) {
+    // Single project mode
+    const projectMeta = await getProjectMeta(client, projectId);
+    const projectMap = await pullSingleProject(client, projectId, projectMeta.idn, verbose);
+    
+    const idMap: ProjectMap = { projects: { [projectMeta.idn]: projectMap } };
+    await fs.writeJson(MAP_PATH, idMap, { spaces: 2 });
+    
+    // Generate hash tracking for this project
+    const hashes: HashStore = {};
+    for (const [agentIdn, agentObj] of Object.entries(projectMap.agents)) {
+      for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+        for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+          const p = skillPath(projectMeta.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+          const content = await fs.readFile(p, 'utf8');
+          hashes[p] = sha256(content);
+        }
+      }
+    }
+    await saveHashes(hashes);
+    return;
+  }
+
+  // Multi-project mode
+  if (verbose) console.log('🔍 Fetching all projects...');
+  const projects = await listProjects(client);
+  if (verbose) console.log(`📦 Found ${projects.length} projects`);
+
+  const idMap: ProjectMap = { projects: {} };
+  const allHashes: HashStore = {};
+
+  for (const project of projects) {
+    if (verbose) console.log(`\n📁 Processing project: ${project.idn} (${project.title})`);
+    const projectMap = await pullSingleProject(client, project.id, project.idn, verbose);
+    idMap.projects[project.idn] = projectMap;
+
+    // Collect hashes for this project
+    for (const [agentIdn, agentObj] of Object.entries(projectMap.agents)) {
+      for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+        for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+          const p = skillPath(project.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+          const content = await fs.readFile(p, 'utf8');
+          allHashes[p] = sha256(content);
+        }
+      }
+    }
+  }
+
+  await fs.writeJson(MAP_PATH, idMap, { spaces: 2 });
+  await saveHashes(allHashes);
+}
+
+export async function pushChanged(client: AxiosInstance, verbose: boolean = false): Promise<void> {
+  await ensureState();
+  if (!(await fs.pathExists(MAP_PATH))) {
+    throw new Error('Missing .newo/map.json. Run `newo pull` first.');
+  }
+  
+  if (verbose) console.log('📋 Loading project mapping...');
+  const idMap = await fs.readJson(MAP_PATH) as ProjectMap | LegacyProjectMap;
+  if (verbose) console.log('🔍 Loading file hashes...');
+  const oldHashes = await loadHashes();
+  const newHashes: HashStore = { ...oldHashes };
+
+  if (verbose) console.log('🔄 Scanning for changes...');
+  let pushed = 0;
+  let scanned = 0;
+  
+  // Handle both old single-project format and new multi-project format
+  const projects = 'projects' in idMap && idMap.projects ? idMap.projects : { '': idMap as ProjectData };
+  
+  for (const [projectIdn, projectData] of Object.entries(projects)) {
+    if (verbose && projectIdn) console.log(`📁 Scanning project: ${projectIdn}`);
+    
+    for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
+      if (verbose) console.log(`  📁 Scanning agent: ${agentIdn}`);
+      for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+        if (verbose) console.log(`    📁 Scanning flow: ${flowIdn}`);
+        for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+          const p = projectIdn ? 
+            skillPath(projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
+            skillPath('', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+          scanned++;
+          if (verbose) console.log(`      📄 Checking: ${p}`);
+          
+          const content = await readIfExists(p);
+          if (content === null) {
+            if (verbose) console.log(`        ⚠️  File not found: ${p}`);
+            continue;
+          }
+          
+          const h = sha256(content);
+          const oldHash = oldHashes[p];
+          if (verbose) {
+            console.log(`        🔍 Hash comparison:`);
+            console.log(`          Old: ${oldHash || 'none'}`);
+            console.log(`          New: ${h}`);
+          }
+          
+          if (oldHash !== h) {
+            if (verbose) console.log(`        🔄 File changed, preparing to push...`);
+            
+            // Create complete skill object with updated prompt_script
+            const skillObject = {
+              id: skillMeta.id,
+              title: skillMeta.title,
+              idn: skillMeta.idn,
+              prompt_script: content,
+              runner_type: skillMeta.runner_type,
+              model: skillMeta.model,
+              parameters: skillMeta.parameters,
+              path: skillMeta.path || undefined
+            };
+            
+            if (verbose) {
+              console.log(`        📤 Pushing skill object:`);
+              console.log(`          ID: ${skillObject.id}`);
+              console.log(`          Title: ${skillObject.title}`);
+              console.log(`          IDN: ${skillObject.idn}`);
+              console.log(`          Content length: ${content.length} chars`);
+              console.log(`          Content preview: ${content.substring(0, 100).replace(/\n/g, '\\n')}...`);
+            }
+            
+            await updateSkill(client, skillObject);
+            console.log(`↑ Pushed ${p}`);
+            newHashes[p] = h;
+            pushed++;
+          } else if (verbose) {
+            console.log(`        ✓ No changes`);
+          }
+        }
+      }
+    }
+  }
+
+  if (verbose) console.log(`🔄 Scanned ${scanned} files, found ${pushed} changes`);
+  await saveHashes(newHashes);
+  console.log(pushed ? `✅ Push complete. ${pushed} file(s) updated.` : '✅ Nothing to push.');
+}
+
+export async function status(verbose: boolean = false): Promise<void> {
+  await ensureState();
+  if (!(await fs.pathExists(MAP_PATH))) {
+    console.log('No map. Run `newo pull` first.');
+    return;
+  }
+  
+  if (verbose) console.log('📋 Loading project mapping and hashes...');
+  const idMap = await fs.readJson(MAP_PATH) as ProjectMap | LegacyProjectMap;
+  const hashes = await loadHashes();
+  let dirty = 0;
+
+  // Handle both old single-project format and new multi-project format
+  const projects = 'projects' in idMap && idMap.projects ? idMap.projects : { '': idMap as ProjectData };
+
+  for (const [projectIdn, projectData] of Object.entries(projects)) {
+    if (verbose && projectIdn) console.log(`📁 Checking project: ${projectIdn}`);
+    
+    for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
+      if (verbose) console.log(`  📁 Checking agent: ${agentIdn}`);
+      for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+        if (verbose) console.log(`    📁 Checking flow: ${flowIdn}`);
+        for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+          const p = projectIdn ? 
+            skillPath(projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
+            skillPath('', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+          const exists = await fs.pathExists(p);
+          if (!exists) { 
+            console.log(`D  ${p}`); 
+            dirty++; 
+            if (verbose) console.log(`      ❌ Deleted: ${p}`);
+            continue; 
+          }
+          const content = await fs.readFile(p, 'utf8');
+          const h = sha256(content);
+          const oldHash = hashes[p];
+          if (verbose) {
+            console.log(`      📄 ${p}`);
+            console.log(`        Old hash: ${oldHash || 'none'}`);
+            console.log(`        New hash: ${h}`);
+          }
+          if (oldHash !== h) { 
+            console.log(`M  ${p}`); 
+            dirty++; 
+            if (verbose) console.log(`      🔄 Modified: ${p}`);
+          } else if (verbose) {
+            console.log(`      ✓ Unchanged: ${p}`);
+          }
+        }
+      }
+    }
+  }
+  console.log(dirty ? `${dirty} changed file(s).` : 'Clean.');
+}
+
+async function generateFlowsYaml(
+  client: AxiosInstance, 
+  agents: Agent[], 
+  projectIdn: string, 
+  verbose: boolean = false
+): Promise<void> {
+  const flowsData: FlowsYamlData = { flows: [] };
+
+  for (const agent of agents) {
+    if (verbose) console.log(`  📁 Processing agent: ${agent.idn}`);
+    
+    const agentFlows: FlowsYamlFlow[] = [];
+    
+    for (const flow of agent.flows ?? []) {
+      if (verbose) console.log(`    📄 Processing flow: ${flow.idn}`);
+      
+      // Get skills for this flow
+      const skills = await listFlowSkills(client, flow.id);
+      const skillsData: FlowsYamlSkill[] = skills.map(skill => ({
+        idn: skill.idn,
+        title: skill.title || "",
+        prompt_script: `flows/${flow.idn}/${skill.idn}.${skill.runner_type === 'nsl' ? 'jinja' : 'nsl'}`,
+        runner_type: `!enum "RunnerType.${skill.runner_type}"`,
+        model: {
+          model_idn: skill.model.model_idn,
+          provider_idn: skill.model.provider_idn
+        },
+        parameters: skill.parameters.map(param => ({
+          name: param.name,
+          default_value: param.default_value || " "
+        }))
+      }));
+
+      // Get events for this flow
+      let eventsData: FlowsYamlEvent[] = [];
+      try {
+        const events = await listFlowEvents(client, flow.id);
+        eventsData = events.map(event => ({
+          title: event.description,
+          idn: event.idn,
+          skill_selector: `!enum "SkillSelector.${event.skill_selector}"`,
+          skill_idn: event.skill_idn || undefined,
+          state_idn: event.state_idn || undefined,
+          integration_idn: event.integration_idn || undefined,
+          connector_idn: event.connector_idn || undefined,
+          interrupt_mode: `!enum "InterruptMode.${event.interrupt_mode}"`
+        }));
+        if (verbose) console.log(`      📋 Found ${events.length} events`);
+      } catch (error) {
+        if (verbose) console.log(`      ⚠️  No events found for flow ${flow.idn}`);
+      }
+
+      // Get state fields for this flow
+      let stateFieldsData: FlowsYamlState[] = [];
+      try {
+        const states = await listFlowStates(client, flow.id);
+        stateFieldsData = states.map(state => ({
+          title: state.title,
+          idn: state.idn,
+          default_value: state.default_value || undefined,
+          scope: `!enum "StateFieldScope.${state.scope}"`
+        }));
+        if (verbose) console.log(`      📊 Found ${states.length} state fields`);
+      } catch (error) {
+        if (verbose) console.log(`      ⚠️  No state fields found for flow ${flow.idn}`);
+      }
+
+      agentFlows.push({
+        idn: flow.idn,
+        title: flow.title,
+        description: flow.description || null,
+        default_runner_type: `!enum "RunnerType.${flow.default_runner_type}"`,
+        default_provider_idn: flow.default_model.provider_idn,
+        default_model_idn: flow.default_model.model_idn,
+        skills: skillsData,
+        events: eventsData,
+        state_fields: stateFieldsData
+      });
+    }
+
+    const agentData: FlowsYamlAgent = {
+      agent_idn: agent.idn,
+      agent_description: agent.description || undefined,
+      agent_flows: agentFlows
+    };
+    
+    flowsData.flows.push(agentData);
+  }
+
+  // Convert to YAML and write to file with custom enum handling
+  let yamlContent = yaml.dump(flowsData, {
+    indent: 2,
+    lineWidth: -1,
+    noRefs: true,
+    sortKeys: false,
+    quotingType: '"',
+    forceQuotes: false
+  });
+  
+  // Post-process to fix enum formatting
+  yamlContent = yamlContent.replace(/"(!enum "[^"]+")"/g, '$1');
+  
+  const yamlPath = path.join(projectDir(projectIdn), 'flows.yaml');
+  await writeFileAtomic(yamlPath, yamlContent);
+  console.log(`✓ Generated flows.yaml for ${projectIdn}`);
+}