newo 1.3.0 → 1.5.0

package/dist/cli.js

@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+import minimist from 'minimist';
+import dotenv from 'dotenv';
+import { makeClient, getProjectMeta, importAkbArticle } from './api.js';
+import { pullAll, pushChanged, status } from './sync.js';
+import { parseAkbFile, prepareArticlesForImport } from './akb.js';
+import path from 'path';
+dotenv.config();
+const { NEWO_PROJECT_ID } = process.env;
+async function main() {
+    const args = minimist(process.argv.slice(2));
+    const cmd = args._[0];
+    const verbose = Boolean(args.verbose || args.v);
+    if (!cmd || ['help', '-h', '--help'].includes(cmd)) {
+        console.log(`NEWO CLI
+Usage:
+  newo pull                    # download all projects -> ./projects/ OR specific project if NEWO_PROJECT_ID set
+  newo push                    # upload modified *.guidance/*.jinja back to NEWO
+  newo status                  # show modified files
+  newo meta                    # get project metadata (debug, requires NEWO_PROJECT_ID)
+  newo import-akb <file> <persona_id>  # import AKB articles from file
+  
+Flags:
+  --verbose, -v                # enable detailed logging
+  
+Env:
+  NEWO_BASE_URL, NEWO_PROJECT_ID (optional), NEWO_API_KEY, NEWO_REFRESH_URL (optional)
+  
+Notes:
+  - multi-project support: pull downloads all accessible projects or single project based on NEWO_PROJECT_ID
+  - If NEWO_PROJECT_ID is set, pull downloads only that project
+  - If NEWO_PROJECT_ID is not set, pull downloads all projects accessible with your API key
+  - Projects are stored in ./projects/{project-idn}/ folders
+  - Each project folder contains metadata.json and flows.yaml
+`);
+        return;
+    }
+    const client = await makeClient(verbose);
+    if (cmd === 'pull') {
+        // If PROJECT_ID is set, pull single project; otherwise pull all projects
+        await pullAll(client, NEWO_PROJECT_ID || null, verbose);
+    }
+    else if (cmd === 'push') {
+        await pushChanged(client, verbose);
+    }
+    else if (cmd === 'status') {
+        await status(verbose);
+    }
+    else if (cmd === 'meta') {
+        if (!NEWO_PROJECT_ID) {
+            throw new Error('NEWO_PROJECT_ID is not set in env');
+        }
+        const meta = await getProjectMeta(client, NEWO_PROJECT_ID);
+        console.log(JSON.stringify(meta, null, 2));
+    }
+    else if (cmd === 'import-akb') {
+        const akbFile = args._[1];
+        const personaId = args._[2];
+        if (!akbFile || !personaId) {
+            console.error('Usage: newo import-akb <file> <persona_id>');
+            console.error('Example: newo import-akb akb.txt da4550db-2b95-4500-91ff-fb4b60fe7be9');
+            process.exit(1);
+        }
+        const filePath = path.resolve(akbFile);
+        try {
+            if (verbose)
+                console.log(`📖 Parsing AKB file: ${filePath}`);
+            const articles = parseAkbFile(filePath);
+            console.log(`✓ Parsed ${articles.length} articles from ${akbFile}`);
+            if (verbose)
+                console.log(`🔧 Preparing articles for persona: ${personaId}`);
+            const preparedArticles = prepareArticlesForImport(articles, personaId);
+            let successCount = 0;
+            let errorCount = 0;
+            console.log(`📤 Importing ${preparedArticles.length} articles...`);
+            for (const [index, article] of preparedArticles.entries()) {
+                try {
+                    if (verbose) {
+                        console.log(`  [${index + 1}/${preparedArticles.length}] Importing ${article.topic_name}...`);
+                    }
+                    await importAkbArticle(client, article);
+                    successCount++;
+                    if (!verbose)
+                        process.stdout.write('.');
+                }
+                catch (error) {
+                    errorCount++;
+                    const errorMessage = error?.response?.data || error.message;
+                    console.error(`\n❌ Failed to import ${article.topic_name}:`, errorMessage);
+                }
+            }
+            if (!verbose)
+                console.log(''); // new line after dots
+            console.log(`✅ Import complete: ${successCount} successful, ${errorCount} failed`);
+        }
+        catch (error) {
+            console.error('❌ AKB import failed:', error.message);
+            process.exit(1);
+        }
+    }
+    else {
+        console.error('Unknown command:', cmd);
+        process.exit(1);
+    }
+}
+main().catch((error) => {
+    const errorData = 'response' in error ? error?.response?.data : error;
+    console.error(errorData || error);
+    process.exit(1);
+});
+//# sourceMappingURL=cli.js.map

package/dist/fsutil.d.ts

@@ -0,0 +1,12 @@
+import type { RunnerType } from './types.js';
+export declare const ROOT_DIR: string;
+export declare const STATE_DIR: string;
+export declare const MAP_PATH: string;
+export declare const HASHES_PATH: string;
+export declare function ensureState(): Promise<void>;
+export declare function projectDir(projectIdn: string): string;
+export declare function skillPath(projectIdn: string, agentIdn: string, flowIdn: string, skillIdn: string, runnerType?: RunnerType): string;
+export declare function metadataPath(projectIdn: string): string;
+export declare function writeFileAtomic(filepath: string, content: string): Promise<void>;
+export declare function readIfExists(filepath: string): Promise<string | null>;
+//# sourceMappingURL=fsutil.d.ts.map

package/dist/fsutil.js

@@ -0,0 +1,28 @@
+import fs from 'fs-extra';
+import path from 'path';
+export const ROOT_DIR = path.join(process.cwd(), 'projects');
+export const STATE_DIR = path.join(process.cwd(), '.newo');
+export const MAP_PATH = path.join(STATE_DIR, 'map.json');
+export const HASHES_PATH = path.join(STATE_DIR, 'hashes.json');
+export async function ensureState() {
+    await fs.ensureDir(STATE_DIR);
+    await fs.ensureDir(ROOT_DIR);
+}
+export function projectDir(projectIdn) {
+    return path.join(ROOT_DIR, projectIdn);
+}
+export function skillPath(projectIdn, agentIdn, flowIdn, skillIdn, runnerType = 'guidance') {
+    const extension = runnerType === 'nsl' ? '.jinja' : '.guidance';
+    return path.join(ROOT_DIR, projectIdn, agentIdn, flowIdn, `${skillIdn}${extension}`);
+}
+export function metadataPath(projectIdn) {
+    return path.join(ROOT_DIR, projectIdn, 'metadata.json');
+}
+export async function writeFileAtomic(filepath, content) {
+    await fs.ensureDir(path.dirname(filepath));
+    await fs.writeFile(filepath, content, 'utf8');
+}
+export async function readIfExists(filepath) {
+    return (await fs.pathExists(filepath)) ? fs.readFile(filepath, 'utf8') : null;
+}
+//# sourceMappingURL=fsutil.js.map

package/dist/hash.d.ts

@@ -0,0 +1,5 @@
+import type { HashStore } from './types.js';
+export declare function sha256(str: string): string;
+export declare function loadHashes(): Promise<HashStore>;
+export declare function saveHashes(hashes: HashStore): Promise<void>;
+//# sourceMappingURL=hash.d.ts.map

package/dist/hash.js

@@ -0,0 +1,17 @@
+import crypto from 'crypto';
+import fs from 'fs-extra';
+import { ensureState, HASHES_PATH } from './fsutil.js';
+export function sha256(str) {
+    return crypto.createHash('sha256').update(str, 'utf8').digest('hex');
+}
+export async function loadHashes() {
+    await ensureState();
+    if (await fs.pathExists(HASHES_PATH)) {
+        return fs.readJson(HASHES_PATH);
+    }
+    return {};
+}
+export async function saveHashes(hashes) {
+    await fs.writeJson(HASHES_PATH, hashes, { spaces: 2 });
+}
+//# sourceMappingURL=hash.js.map

package/dist/sync.d.ts

@@ -0,0 +1,7 @@
+import type { AxiosInstance } from 'axios';
+import type { ProjectData } from './types.js';
+export declare function pullSingleProject(client: AxiosInstance, projectId: string, projectIdn: string, verbose?: boolean): Promise<ProjectData>;
+export declare function pullAll(client: AxiosInstance, projectId?: string | null, verbose?: boolean): Promise<void>;
+export declare function pushChanged(client: AxiosInstance, verbose?: boolean): Promise<void>;
+export declare function status(verbose?: boolean): Promise<void>;
+//# sourceMappingURL=sync.d.ts.map

package/dist/sync.js

@@ -0,0 +1,337 @@
+import { listProjects, listAgents, listFlowSkills, updateSkill, listFlowEvents, listFlowStates, getProjectMeta } from './api.js';
+import { ensureState, skillPath, writeFileAtomic, readIfExists, MAP_PATH, projectDir, metadataPath } from './fsutil.js';
+import fs from 'fs-extra';
+import { sha256, loadHashes, saveHashes } from './hash.js';
+import yaml from 'js-yaml';
+import path from 'path';
+export async function pullSingleProject(client, projectId, projectIdn, verbose = false) {
+    if (verbose)
+        console.log(`🔍 Fetching agents for project ${projectId} (${projectIdn})...`);
+    const agents = await listAgents(client, projectId);
+    if (verbose)
+        console.log(`📦 Found ${agents.length} agents`);
+    // Get and save project metadata
+    const projectMeta = await getProjectMeta(client, projectId);
+    await writeFileAtomic(metadataPath(projectIdn), JSON.stringify(projectMeta, null, 2));
+    if (verbose)
+        console.log(`✓ Saved metadata for ${projectIdn}`);
+    const projectMap = { projectId, projectIdn, agents: {} };
+    for (const agent of agents) {
+        const aKey = agent.idn;
+        projectMap.agents[aKey] = { id: agent.id, flows: {} };
+        for (const flow of agent.flows ?? []) {
+            projectMap.agents[aKey].flows[flow.idn] = { id: flow.id, skills: {} };
+            const skills = await listFlowSkills(client, flow.id);
+            for (const skill of skills) {
+                const file = skillPath(projectIdn, agent.idn, flow.idn, skill.idn, skill.runner_type);
+                await writeFileAtomic(file, skill.prompt_script || '');
+                // Store complete skill metadata for push operations
+                projectMap.agents[aKey].flows[flow.idn].skills[skill.idn] = {
+                    id: skill.id,
+                    title: skill.title,
+                    idn: skill.idn,
+                    runner_type: skill.runner_type,
+                    model: skill.model,
+                    parameters: skill.parameters,
+                    path: skill.path || undefined
+                };
+                console.log(`✓ Pulled ${file}`);
+            }
+        }
+    }
+    // Generate flows.yaml for this project
+    if (verbose)
+        console.log(`📄 Generating flows.yaml for ${projectIdn}...`);
+    await generateFlowsYaml(client, agents, projectIdn, verbose);
+    return projectMap;
+}
+export async function pullAll(client, projectId = null, verbose = false) {
+    await ensureState();
+    if (projectId) {
+        // Single project mode
+        const projectMeta = await getProjectMeta(client, projectId);
+        const projectMap = await pullSingleProject(client, projectId, projectMeta.idn, verbose);
+        const idMap = { projects: { [projectMeta.idn]: projectMap } };
+        await fs.writeJson(MAP_PATH, idMap, { spaces: 2 });
+        // Generate hash tracking for this project
+        const hashes = {};
+        for (const [agentIdn, agentObj] of Object.entries(projectMap.agents)) {
+            for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+                    const p = skillPath(projectMeta.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+                    const content = await fs.readFile(p, 'utf8');
+                    hashes[p] = sha256(content);
+                }
+            }
+        }
+        await saveHashes(hashes);
+        return;
+    }
+    // Multi-project mode
+    if (verbose)
+        console.log('🔍 Fetching all projects...');
+    const projects = await listProjects(client);
+    if (verbose)
+        console.log(`📦 Found ${projects.length} projects`);
+    const idMap = { projects: {} };
+    const allHashes = {};
+    for (const project of projects) {
+        if (verbose)
+            console.log(`\n📁 Processing project: ${project.idn} (${project.title})`);
+        const projectMap = await pullSingleProject(client, project.id, project.idn, verbose);
+        idMap.projects[project.idn] = projectMap;
+        // Collect hashes for this project
+        for (const [agentIdn, agentObj] of Object.entries(projectMap.agents)) {
+            for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+                    const p = skillPath(project.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+                    const content = await fs.readFile(p, 'utf8');
+                    allHashes[p] = sha256(content);
+                }
+            }
+        }
+    }
+    await fs.writeJson(MAP_PATH, idMap, { spaces: 2 });
+    await saveHashes(allHashes);
+}
+export async function pushChanged(client, verbose = false) {
+    await ensureState();
+    if (!(await fs.pathExists(MAP_PATH))) {
+        throw new Error('Missing .newo/map.json. Run `newo pull` first.');
+    }
+    if (verbose)
+        console.log('📋 Loading project mapping...');
+    const idMap = await fs.readJson(MAP_PATH);
+    if (verbose)
+        console.log('🔍 Loading file hashes...');
+    const oldHashes = await loadHashes();
+    const newHashes = { ...oldHashes };
+    if (verbose)
+        console.log('🔄 Scanning for changes...');
+    let pushed = 0;
+    let scanned = 0;
+    // Handle both old single-project format and new multi-project format
+    const projects = 'projects' in idMap && idMap.projects ? idMap.projects : { '': idMap };
+    for (const [projectIdn, projectData] of Object.entries(projects)) {
+        if (verbose && projectIdn)
+            console.log(`📁 Scanning project: ${projectIdn}`);
+        for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
+            if (verbose)
+                console.log(`  📁 Scanning agent: ${agentIdn}`);
+            for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                if (verbose)
+                    console.log(`    📁 Scanning flow: ${flowIdn}`);
+                for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+                    const p = projectIdn ?
+                        skillPath(projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
+                        skillPath('', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+                    scanned++;
+                    if (verbose)
+                        console.log(`      📄 Checking: ${p}`);
+                    const content = await readIfExists(p);
+                    if (content === null) {
+                        if (verbose)
+                            console.log(`        ⚠️  File not found: ${p}`);
+                        continue;
+                    }
+                    const h = sha256(content);
+                    const oldHash = oldHashes[p];
+                    if (verbose) {
+                        console.log(`        🔍 Hash comparison:`);
+                        console.log(`          Old: ${oldHash || 'none'}`);
+                        console.log(`          New: ${h}`);
+                    }
+                    if (oldHash !== h) {
+                        if (verbose)
+                            console.log(`        🔄 File changed, preparing to push...`);
+                        // Create complete skill object with updated prompt_script
+                        const skillObject = {
+                            id: skillMeta.id,
+                            title: skillMeta.title,
+                            idn: skillMeta.idn,
+                            prompt_script: content,
+                            runner_type: skillMeta.runner_type,
+                            model: skillMeta.model,
+                            parameters: skillMeta.parameters,
+                            path: skillMeta.path || undefined
+                        };
+                        if (verbose) {
+                            console.log(`        📤 Pushing skill object:`);
+                            console.log(`          ID: ${skillObject.id}`);
+                            console.log(`          Title: ${skillObject.title}`);
+                            console.log(`          IDN: ${skillObject.idn}`);
+                            console.log(`          Content length: ${content.length} chars`);
+                            console.log(`          Content preview: ${content.substring(0, 100).replace(/\n/g, '\\n')}...`);
+                        }
+                        await updateSkill(client, skillObject);
+                        console.log(`↑ Pushed ${p}`);
+                        newHashes[p] = h;
+                        pushed++;
+                    }
+                    else if (verbose) {
+                        console.log(`        ✓ No changes`);
+                    }
+                }
+            }
+        }
+    }
+    if (verbose)
+        console.log(`🔄 Scanned ${scanned} files, found ${pushed} changes`);
+    await saveHashes(newHashes);
+    console.log(pushed ? `✅ Push complete. ${pushed} file(s) updated.` : '✅ Nothing to push.');
+}
+export async function status(verbose = false) {
+    await ensureState();
+    if (!(await fs.pathExists(MAP_PATH))) {
+        console.log('No map. Run `newo pull` first.');
+        return;
+    }
+    if (verbose)
+        console.log('📋 Loading project mapping and hashes...');
+    const idMap = await fs.readJson(MAP_PATH);
+    const hashes = await loadHashes();
+    let dirty = 0;
+    // Handle both old single-project format and new multi-project format
+    const projects = 'projects' in idMap && idMap.projects ? idMap.projects : { '': idMap };
+    for (const [projectIdn, projectData] of Object.entries(projects)) {
+        if (verbose && projectIdn)
+            console.log(`📁 Checking project: ${projectIdn}`);
+        for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
+            if (verbose)
+                console.log(`  📁 Checking agent: ${agentIdn}`);
+            for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                if (verbose)
+                    console.log(`    📁 Checking flow: ${flowIdn}`);
+                for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+                    const p = projectIdn ?
+                        skillPath(projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
+                        skillPath('', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+                    const exists = await fs.pathExists(p);
+                    if (!exists) {
+                        console.log(`D  ${p}`);
+                        dirty++;
+                        if (verbose)
+                            console.log(`      ❌ Deleted: ${p}`);
+                        continue;
+                    }
+                    const content = await fs.readFile(p, 'utf8');
+                    const h = sha256(content);
+                    const oldHash = hashes[p];
+                    if (verbose) {
+                        console.log(`      📄 ${p}`);
+                        console.log(`        Old hash: ${oldHash || 'none'}`);
+                        console.log(`        New hash: ${h}`);
+                    }
+                    if (oldHash !== h) {
+                        console.log(`M  ${p}`);
+                        dirty++;
+                        if (verbose)
+                            console.log(`      🔄 Modified: ${p}`);
+                    }
+                    else if (verbose) {
+                        console.log(`      ✓ Unchanged: ${p}`);
+                    }
+                }
+            }
+        }
+    }
+    console.log(dirty ? `${dirty} changed file(s).` : 'Clean.');
+}
+async function generateFlowsYaml(client, agents, projectIdn, verbose = false) {
+    const flowsData = { flows: [] };
+    for (const agent of agents) {
+        if (verbose)
+            console.log(`  📁 Processing agent: ${agent.idn}`);
+        const agentFlows = [];
+        for (const flow of agent.flows ?? []) {
+            if (verbose)
+                console.log(`    📄 Processing flow: ${flow.idn}`);
+            // Get skills for this flow
+            const skills = await listFlowSkills(client, flow.id);
+            const skillsData = skills.map(skill => ({
+                idn: skill.idn,
+                title: skill.title || "",
+                prompt_script: `flows/${flow.idn}/${skill.idn}.${skill.runner_type === 'nsl' ? 'jinja' : 'nsl'}`,
+                runner_type: `!enum "RunnerType.${skill.runner_type}"`,
+                model: {
+                    model_idn: skill.model.model_idn,
+                    provider_idn: skill.model.provider_idn
+                },
+                parameters: skill.parameters.map(param => ({
+                    name: param.name,
+                    default_value: param.default_value || " "
+                }))
+            }));
+            // Get events for this flow
+            let eventsData = [];
+            try {
+                const events = await listFlowEvents(client, flow.id);
+                eventsData = events.map(event => ({
+                    title: event.description,
+                    idn: event.idn,
+                    skill_selector: `!enum "SkillSelector.${event.skill_selector}"`,
+                    skill_idn: event.skill_idn || undefined,
+                    state_idn: event.state_idn || undefined,
+                    integration_idn: event.integration_idn || undefined,
+                    connector_idn: event.connector_idn || undefined,
+                    interrupt_mode: `!enum "InterruptMode.${event.interrupt_mode}"`
+                }));
+                if (verbose)
+                    console.log(`      📋 Found ${events.length} events`);
+            }
+            catch (error) {
+                if (verbose)
+                    console.log(`      ⚠️  No events found for flow ${flow.idn}`);
+            }
+            // Get state fields for this flow
+            let stateFieldsData = [];
+            try {
+                const states = await listFlowStates(client, flow.id);
+                stateFieldsData = states.map(state => ({
+                    title: state.title,
+                    idn: state.idn,
+                    default_value: state.default_value || undefined,
+                    scope: `!enum "StateFieldScope.${state.scope}"`
+                }));
+                if (verbose)
+                    console.log(`      📊 Found ${states.length} state fields`);
+            }
+            catch (error) {
+                if (verbose)
+                    console.log(`      ⚠️  No state fields found for flow ${flow.idn}`);
+            }
+            agentFlows.push({
+                idn: flow.idn,
+                title: flow.title,
+                description: flow.description || null,
+                default_runner_type: `!enum "RunnerType.${flow.default_runner_type}"`,
+                default_provider_idn: flow.default_model.provider_idn,
+                default_model_idn: flow.default_model.model_idn,
+                skills: skillsData,
+                events: eventsData,
+                state_fields: stateFieldsData
+            });
+        }
+        const agentData = {
+            agent_idn: agent.idn,
+            agent_description: agent.description || undefined,
+            agent_flows: agentFlows
+        };
+        flowsData.flows.push(agentData);
+    }
+    // Convert to YAML and write to file with custom enum handling
+    let yamlContent = yaml.dump(flowsData, {
+        indent: 2,
+        lineWidth: -1,
+        noRefs: true,
+        sortKeys: false,
+        quotingType: '"',
+        forceQuotes: false
+    });
+    // Post-process to fix enum formatting
+    yamlContent = yamlContent.replace(/"(!enum "[^"]+")"/g, '$1');
+    const yamlPath = path.join(projectDir(projectIdn), 'flows.yaml');
+    await writeFileAtomic(yamlPath, yamlContent);
+    console.log(`✓ Generated flows.yaml for ${projectIdn}`);
+}
+//# sourceMappingURL=sync.js.map