newo 1.4.0 → 1.5.0

package/dist/sync.js

@@ -0,0 +1,337 @@
+import { listProjects, listAgents, listFlowSkills, updateSkill, listFlowEvents, listFlowStates, getProjectMeta } from './api.js';
+import { ensureState, skillPath, writeFileAtomic, readIfExists, MAP_PATH, projectDir, metadataPath } from './fsutil.js';
+import fs from 'fs-extra';
+import { sha256, loadHashes, saveHashes } from './hash.js';
+import yaml from 'js-yaml';
+import path from 'path';
+export async function pullSingleProject(client, projectId, projectIdn, verbose = false) {
+    if (verbose)
+        console.log(`🔍 Fetching agents for project ${projectId} (${projectIdn})...`);
+    const agents = await listAgents(client, projectId);
+    if (verbose)
+        console.log(`📦 Found ${agents.length} agents`);
+    // Get and save project metadata
+    const projectMeta = await getProjectMeta(client, projectId);
+    await writeFileAtomic(metadataPath(projectIdn), JSON.stringify(projectMeta, null, 2));
+    if (verbose)
+        console.log(`✓ Saved metadata for ${projectIdn}`);
+    const projectMap = { projectId, projectIdn, agents: {} };
+    for (const agent of agents) {
+        const aKey = agent.idn;
+        projectMap.agents[aKey] = { id: agent.id, flows: {} };
+        for (const flow of agent.flows ?? []) {
+            projectMap.agents[aKey].flows[flow.idn] = { id: flow.id, skills: {} };
+            const skills = await listFlowSkills(client, flow.id);
+            for (const skill of skills) {
+                const file = skillPath(projectIdn, agent.idn, flow.idn, skill.idn, skill.runner_type);
+                await writeFileAtomic(file, skill.prompt_script || '');
+                // Store complete skill metadata for push operations
+                projectMap.agents[aKey].flows[flow.idn].skills[skill.idn] = {
+                    id: skill.id,
+                    title: skill.title,
+                    idn: skill.idn,
+                    runner_type: skill.runner_type,
+                    model: skill.model,
+                    parameters: skill.parameters,
+                    path: skill.path || undefined
+                };
+                console.log(`✓ Pulled ${file}`);
+            }
+        }
+    }
+    // Generate flows.yaml for this project
+    if (verbose)
+        console.log(`📄 Generating flows.yaml for ${projectIdn}...`);
+    await generateFlowsYaml(client, agents, projectIdn, verbose);
+    return projectMap;
+}
+export async function pullAll(client, projectId = null, verbose = false) {
+    await ensureState();
+    if (projectId) {
+        // Single project mode
+        const projectMeta = await getProjectMeta(client, projectId);
+        const projectMap = await pullSingleProject(client, projectId, projectMeta.idn, verbose);
+        const idMap = { projects: { [projectMeta.idn]: projectMap } };
+        await fs.writeJson(MAP_PATH, idMap, { spaces: 2 });
+        // Generate hash tracking for this project
+        const hashes = {};
+        for (const [agentIdn, agentObj] of Object.entries(projectMap.agents)) {
+            for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+                    const p = skillPath(projectMeta.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+                    const content = await fs.readFile(p, 'utf8');
+                    hashes[p] = sha256(content);
+                }
+            }
+        }
+        await saveHashes(hashes);
+        return;
+    }
+    // Multi-project mode
+    if (verbose)
+        console.log('🔍 Fetching all projects...');
+    const projects = await listProjects(client);
+    if (verbose)
+        console.log(`📦 Found ${projects.length} projects`);
+    const idMap = { projects: {} };
+    const allHashes = {};
+    for (const project of projects) {
+        if (verbose)
+            console.log(`\n📁 Processing project: ${project.idn} (${project.title})`);
+        const projectMap = await pullSingleProject(client, project.id, project.idn, verbose);
+        idMap.projects[project.idn] = projectMap;
+        // Collect hashes for this project
+        for (const [agentIdn, agentObj] of Object.entries(projectMap.agents)) {
+            for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+                    const p = skillPath(project.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+                    const content = await fs.readFile(p, 'utf8');
+                    allHashes[p] = sha256(content);
+                }
+            }
+        }
+    }
+    await fs.writeJson(MAP_PATH, idMap, { spaces: 2 });
+    await saveHashes(allHashes);
+}
+export async function pushChanged(client, verbose = false) {
+    await ensureState();
+    if (!(await fs.pathExists(MAP_PATH))) {
+        throw new Error('Missing .newo/map.json. Run `newo pull` first.');
+    }
+    if (verbose)
+        console.log('📋 Loading project mapping...');
+    const idMap = await fs.readJson(MAP_PATH);
+    if (verbose)
+        console.log('🔍 Loading file hashes...');
+    const oldHashes = await loadHashes();
+    const newHashes = { ...oldHashes };
+    if (verbose)
+        console.log('🔄 Scanning for changes...');
+    let pushed = 0;
+    let scanned = 0;
+    // Handle both old single-project format and new multi-project format
+    const projects = 'projects' in idMap && idMap.projects ? idMap.projects : { '': idMap };
+    for (const [projectIdn, projectData] of Object.entries(projects)) {
+        if (verbose && projectIdn)
+            console.log(`📁 Scanning project: ${projectIdn}`);
+        for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
+            if (verbose)
+                console.log(`  📁 Scanning agent: ${agentIdn}`);
+            for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                if (verbose)
+                    console.log(`    📁 Scanning flow: ${flowIdn}`);
+                for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+                    const p = projectIdn ?
+                        skillPath(projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
+                        skillPath('', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+                    scanned++;
+                    if (verbose)
+                        console.log(`      📄 Checking: ${p}`);
+                    const content = await readIfExists(p);
+                    if (content === null) {
+                        if (verbose)
+                            console.log(`        ⚠️  File not found: ${p}`);
+                        continue;
+                    }
+                    const h = sha256(content);
+                    const oldHash = oldHashes[p];
+                    if (verbose) {
+                        console.log(`        🔍 Hash comparison:`);
+                        console.log(`          Old: ${oldHash || 'none'}`);
+                        console.log(`          New: ${h}`);
+                    }
+                    if (oldHash !== h) {
+                        if (verbose)
+                            console.log(`        🔄 File changed, preparing to push...`);
+                        // Create complete skill object with updated prompt_script
+                        const skillObject = {
+                            id: skillMeta.id,
+                            title: skillMeta.title,
+                            idn: skillMeta.idn,
+                            prompt_script: content,
+                            runner_type: skillMeta.runner_type,
+                            model: skillMeta.model,
+                            parameters: skillMeta.parameters,
+                            path: skillMeta.path || undefined
+                        };
+                        if (verbose) {
+                            console.log(`        📤 Pushing skill object:`);
+                            console.log(`          ID: ${skillObject.id}`);
+                            console.log(`          Title: ${skillObject.title}`);
+                            console.log(`          IDN: ${skillObject.idn}`);
+                            console.log(`          Content length: ${content.length} chars`);
+                            console.log(`          Content preview: ${content.substring(0, 100).replace(/\n/g, '\\n')}...`);
+                        }
+                        await updateSkill(client, skillObject);
+                        console.log(`↑ Pushed ${p}`);
+                        newHashes[p] = h;
+                        pushed++;
+                    }
+                    else if (verbose) {
+                        console.log(`        ✓ No changes`);
+                    }
+                }
+            }
+        }
+    }
+    if (verbose)
+        console.log(`🔄 Scanned ${scanned} files, found ${pushed} changes`);
+    await saveHashes(newHashes);
+    console.log(pushed ? `✅ Push complete. ${pushed} file(s) updated.` : '✅ Nothing to push.');
+}
+export async function status(verbose = false) {
+    await ensureState();
+    if (!(await fs.pathExists(MAP_PATH))) {
+        console.log('No map. Run `newo pull` first.');
+        return;
+    }
+    if (verbose)
+        console.log('📋 Loading project mapping and hashes...');
+    const idMap = await fs.readJson(MAP_PATH);
+    const hashes = await loadHashes();
+    let dirty = 0;
+    // Handle both old single-project format and new multi-project format
+    const projects = 'projects' in idMap && idMap.projects ? idMap.projects : { '': idMap };
+    for (const [projectIdn, projectData] of Object.entries(projects)) {
+        if (verbose && projectIdn)
+            console.log(`📁 Checking project: ${projectIdn}`);
+        for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
+            if (verbose)
+                console.log(`  📁 Checking agent: ${agentIdn}`);
+            for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+                if (verbose)
+                    console.log(`    📁 Checking flow: ${flowIdn}`);
+                for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
+                    const p = projectIdn ?
+                        skillPath(projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
+                        skillPath('', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
+                    const exists = await fs.pathExists(p);
+                    if (!exists) {
+                        console.log(`D  ${p}`);
+                        dirty++;
+                        if (verbose)
+                            console.log(`      ❌ Deleted: ${p}`);
+                        continue;
+                    }
+                    const content = await fs.readFile(p, 'utf8');
+                    const h = sha256(content);
+                    const oldHash = hashes[p];
+                    if (verbose) {
+                        console.log(`      📄 ${p}`);
+                        console.log(`        Old hash: ${oldHash || 'none'}`);
+                        console.log(`        New hash: ${h}`);
+                    }
+                    if (oldHash !== h) {
+                        console.log(`M  ${p}`);
+                        dirty++;
+                        if (verbose)
+                            console.log(`      🔄 Modified: ${p}`);
+                    }
+                    else if (verbose) {
+                        console.log(`      ✓ Unchanged: ${p}`);
+                    }
+                }
+            }
+        }
+    }
+    console.log(dirty ? `${dirty} changed file(s).` : 'Clean.');
+}
+async function generateFlowsYaml(client, agents, projectIdn, verbose = false) {
+    const flowsData = { flows: [] };
+    for (const agent of agents) {
+        if (verbose)
+            console.log(`  📁 Processing agent: ${agent.idn}`);
+        const agentFlows = [];
+        for (const flow of agent.flows ?? []) {
+            if (verbose)
+                console.log(`    📄 Processing flow: ${flow.idn}`);
+            // Get skills for this flow
+            const skills = await listFlowSkills(client, flow.id);
+            const skillsData = skills.map(skill => ({
+                idn: skill.idn,
+                title: skill.title || "",
+                prompt_script: `flows/${flow.idn}/${skill.idn}.${skill.runner_type === 'nsl' ? 'jinja' : 'nsl'}`,
+                runner_type: `!enum "RunnerType.${skill.runner_type}"`,
+                model: {
+                    model_idn: skill.model.model_idn,
+                    provider_idn: skill.model.provider_idn
+                },
+                parameters: skill.parameters.map(param => ({
+                    name: param.name,
+                    default_value: param.default_value || " "
+                }))
+            }));
+            // Get events for this flow
+            let eventsData = [];
+            try {
+                const events = await listFlowEvents(client, flow.id);
+                eventsData = events.map(event => ({
+                    title: event.description,
+                    idn: event.idn,
+                    skill_selector: `!enum "SkillSelector.${event.skill_selector}"`,
+                    skill_idn: event.skill_idn || undefined,
+                    state_idn: event.state_idn || undefined,
+                    integration_idn: event.integration_idn || undefined,
+                    connector_idn: event.connector_idn || undefined,
+                    interrupt_mode: `!enum "InterruptMode.${event.interrupt_mode}"`
+                }));
+                if (verbose)
+                    console.log(`      📋 Found ${events.length} events`);
+            }
+            catch (error) {
+                if (verbose)
+                    console.log(`      ⚠️  No events found for flow ${flow.idn}`);
+            }
+            // Get state fields for this flow
+            let stateFieldsData = [];
+            try {
+                const states = await listFlowStates(client, flow.id);
+                stateFieldsData = states.map(state => ({
+                    title: state.title,
+                    idn: state.idn,
+                    default_value: state.default_value || undefined,
+                    scope: `!enum "StateFieldScope.${state.scope}"`
+                }));
+                if (verbose)
+                    console.log(`      📊 Found ${states.length} state fields`);
+            }
+            catch (error) {
+                if (verbose)
+                    console.log(`      ⚠️  No state fields found for flow ${flow.idn}`);
+            }
+            agentFlows.push({
+                idn: flow.idn,
+                title: flow.title,
+                description: flow.description || null,
+                default_runner_type: `!enum "RunnerType.${flow.default_runner_type}"`,
+                default_provider_idn: flow.default_model.provider_idn,
+                default_model_idn: flow.default_model.model_idn,
+                skills: skillsData,
+                events: eventsData,
+                state_fields: stateFieldsData
+            });
+        }
+        const agentData = {
+            agent_idn: agent.idn,
+            agent_description: agent.description || undefined,
+            agent_flows: agentFlows
+        };
+        flowsData.flows.push(agentData);
+    }
+    // Convert to YAML and write to file with custom enum handling
+    let yamlContent = yaml.dump(flowsData, {
+        indent: 2,
+        lineWidth: -1,
+        noRefs: true,
+        sortKeys: false,
+        quotingType: '"',
+        forceQuotes: false
+    });
+    // Post-process to fix enum formatting
+    yamlContent = yamlContent.replace(/"(!enum "[^"]+")"/g, '$1');
+    const yamlPath = path.join(projectDir(projectIdn), 'flows.yaml');
+    await writeFileAtomic(yamlPath, yamlContent);
+    console.log(`✓ Generated flows.yaml for ${projectIdn}`);
+}
+//# sourceMappingURL=sync.js.map

package/dist/types.d.ts

@@ -0,0 +1,206 @@
+/**
+ * Comprehensive type definitions for NEWO CLI
+ */
+export interface NewoEnvironment {
+    NEWO_BASE_URL?: string;
+    NEWO_PROJECT_ID?: string;
+    NEWO_API_KEY?: string;
+    NEWO_ACCESS_TOKEN?: string;
+    NEWO_REFRESH_TOKEN?: string;
+    NEWO_REFRESH_URL?: string;
+}
+export interface TokenResponse {
+    access_token?: string;
+    token?: string;
+    accessToken?: string;
+    refresh_token?: string;
+    refreshToken?: string;
+    expires_in?: number;
+    expiresIn?: number;
+}
+export interface StoredTokens {
+    access_token: string;
+    refresh_token: string;
+    expires_at: number;
+}
+export interface ProjectMeta {
+    id: string;
+    idn: string;
+    title: string;
+    description?: string;
+    created_at?: string;
+    updated_at?: string;
+}
+export interface Agent {
+    id: string;
+    idn: string;
+    title?: string;
+    description?: string;
+    flows?: Flow[];
+}
+export interface Flow {
+    id: string;
+    idn: string;
+    title: string;
+    description?: string;
+    default_runner_type: RunnerType;
+    default_model: ModelConfig;
+}
+export interface ModelConfig {
+    model_idn: string;
+    provider_idn: string;
+}
+export interface SkillParameter {
+    name: string;
+    default_value?: string;
+}
+export interface Skill {
+    id: string;
+    idn: string;
+    title: string;
+    prompt_script?: string;
+    runner_type: RunnerType;
+    model: ModelConfig;
+    parameters: SkillParameter[];
+    path?: string | undefined;
+}
+export interface FlowEvent {
+    id: string;
+    idn: string;
+    description: string;
+    skill_selector: SkillSelector;
+    skill_idn?: string;
+    state_idn?: string;
+    integration_idn?: string;
+    connector_idn?: string;
+    interrupt_mode: InterruptMode;
+}
+export interface FlowState {
+    id: string;
+    idn: string;
+    title: string;
+    default_value?: string;
+    scope: StateFieldScope;
+}
+export type RunnerType = 'guidance' | 'nsl';
+export type SkillSelector = 'first' | 'last' | 'random' | 'all';
+export type InterruptMode = 'allow' | 'deny' | 'queue';
+export type StateFieldScope = 'flow' | 'agent' | 'project' | 'global';
+export interface SkillMetadata {
+    id: string;
+    title: string;
+    idn: string;
+    runner_type: RunnerType;
+    model: ModelConfig;
+    parameters: SkillParameter[];
+    path?: string | undefined;
+}
+export interface FlowData {
+    id: string;
+    skills: Record<string, SkillMetadata>;
+}
+export interface AgentData {
+    id: string;
+    flows: Record<string, FlowData>;
+}
+export interface ProjectData {
+    projectId: string;
+    projectIdn: string;
+    agents: Record<string, AgentData>;
+}
+export interface ProjectMap {
+    projects: Record<string, ProjectData>;
+}
+export interface LegacyProjectMap extends ProjectData {
+    projects?: Record<string, ProjectData>;
+}
+export interface HashStore {
+    [filePath: string]: string;
+}
+export interface ParsedArticle {
+    topic_name: string;
+    persona_id: string | null;
+    topic_summary: string;
+    topic_facts: string[];
+    confidence: number;
+    source: string;
+    labels: string[];
+}
+export interface AkbImportArticle extends Omit<ParsedArticle, 'persona_id'> {
+    persona_id: string;
+}
+export interface CliArgs {
+    _: string[];
+    verbose?: boolean;
+    v?: boolean;
+    [key: string]: unknown;
+}
+export interface FlowsYamlSkill {
+    idn: string;
+    title: string;
+    prompt_script: string;
+    runner_type: string;
+    model: ModelConfig;
+    parameters: Array<{
+        name: string;
+        default_value: string;
+    }>;
+}
+export interface FlowsYamlEvent {
+    title: string;
+    idn: string;
+    skill_selector: string;
+    skill_idn?: string | undefined;
+    state_idn?: string | undefined;
+    integration_idn?: string | undefined;
+    connector_idn?: string | undefined;
+    interrupt_mode: string;
+}
+export interface FlowsYamlState {
+    title: string;
+    idn: string;
+    default_value?: string | undefined;
+    scope: string;
+}
+export interface FlowsYamlFlow {
+    idn: string;
+    title: string;
+    description: string | null;
+    default_runner_type: string;
+    default_provider_idn: string;
+    default_model_idn: string;
+    skills: FlowsYamlSkill[];
+    events: FlowsYamlEvent[];
+    state_fields: FlowsYamlState[];
+}
+export interface FlowsYamlAgent {
+    agent_idn: string;
+    agent_description?: string | undefined;
+    agent_flows: FlowsYamlFlow[];
+}
+export interface FlowsYamlData {
+    flows: FlowsYamlAgent[];
+}
+export interface AxiosClientConfig {
+    baseURL?: string;
+    headers?: Record<string, string>;
+}
+export interface NewoApiError extends Error {
+    response?: {
+        status: number;
+        data: unknown;
+    };
+    config?: {
+        method?: string;
+        url?: string;
+        headers?: Record<string, string>;
+    };
+}
+export type FileStatus = 'M' | 'D' | 'clean';
+export interface StatusResult {
+    filePath: string;
+    status: FileStatus;
+    oldHash?: string;
+    newHash?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.js

@@ -0,0 +1,5 @@
+/**
+ * Comprehensive type definitions for NEWO CLI
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "newo",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "NEWO CLI: sync flows/skills between NEWO and local files, multi-project support, import AKB articles",
   "type": "module",
   "bin": {
-    "newo": "src/cli.js"
+    "newo": "dist/cli.js"
   },
   "files": [
-    "src/**/*.js",
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "src/**/*.ts",
     "README.md",
     "CHANGELOG.md",
     ".env.example"
@@ -47,16 +49,28 @@
     "minimist": "^1.2.8"
   },
   "devDependencies": {
-    "mocha": "^10.2.0"
+    "@types/fs-extra": "^11.0.4",
+    "@types/js-yaml": "^4.0.9",
+    "@types/minimist": "^1.2.5",
+    "@types/node": "^22.5.4",
+    "mocha": "^10.2.0",
+    "typescript": "^5.6.2"
   },
   "scripts": {
-    "dev": "node ./src/cli.js",
-    "pull": "node ./src/cli.js pull",
-    "push": "node ./src/cli.js push",
-    "status": "node ./src/cli.js status",
-    "test": "mocha test/*.test.js --timeout 60000",
-    "test:api": "mocha test/api.test.js --timeout 30000",
-    "test:sync": "mocha test/sync.test.js --timeout 60000",
-    "test:integration": "mocha test/integration.test.js --timeout 120000"
+    "build": "tsc",
+    "build:watch": "tsc --watch",
+    "dev": "npm run build && node ./dist/cli.js",
+    "dev:watch": "tsc --watch & nodemon --watch dist dist/cli.js",
+    "pull": "npm run build && node ./dist/cli.js pull",
+    "push": "npm run build && node ./dist/cli.js push",
+    "status": "npm run build && node ./dist/cli.js status",
+    "clean": "rm -rf dist",
+    "typecheck": "tsc --noEmit",
+    "lint": "tsc --noEmit --strict",
+    "test": "npm run build && mocha test/*.test.js --timeout 60000",
+    "test:api": "npm run build && mocha test/api.test.js --timeout 30000",
+    "test:sync": "npm run build && mocha test/sync.test.js --timeout 60000",
+    "test:integration": "npm run build && mocha test/integration.test.js --timeout 120000",
+    "prepublishOnly": "npm run clean && npm run build"
   }
 }