newo 1.9.2 → 2.0.0

package/dist/sync/attributes.js

@@ -0,0 +1,90 @@
+/**
+ * Customer attributes synchronization module
+ */
+import { getCustomerAttributes } from '../api.js';
+import { writeFileSafe, customerAttributesPath, customerAttributesMapPath, customerAttributesBackupPath } from '../fsutil.js';
+import yaml from 'js-yaml';
+/**
+ * Save customer attributes to YAML format and return content for hashing
+ */
+export async function saveCustomerAttributes(client, customer, verbose = false) {
+    if (verbose)
+        console.log(`🔍 Fetching customer attributes for ${customer.idn}...`);
+    try {
+        const response = await getCustomerAttributes(client, true); // Include hidden attributes
+        // API returns { groups: [...], attributes: [...] }
+        // We only want the attributes array in the expected format
+        const attributes = response.attributes || response;
+        if (verbose)
+            console.log(`📦 Found ${Array.isArray(attributes) ? attributes.length : 'invalid'} attributes`);
+        // Create ID mapping for push operations (separate from YAML)
+        const idMapping = {};
+        // Transform attributes to match reference format exactly (no ID fields)
+        const cleanAttributes = Array.isArray(attributes) ? attributes.map(attr => {
+            // Store ID mapping for push operations
+            if (attr.id) {
+                idMapping[attr.idn] = attr.id;
+            }
+            // Special handling for complex JSON string values
+            let processedValue = attr.value;
+            if (typeof attr.value === 'string' && attr.value.startsWith('[{') && attr.value.endsWith('}]')) {
+                try {
+                    // Parse and reformat JSON for better readability
+                    const parsed = JSON.parse(attr.value);
+                    processedValue = JSON.stringify(parsed, null, 0); // No extra spacing, but valid JSON
+                }
+                catch (e) {
+                    // Keep original if parsing fails
+                    processedValue = attr.value;
+                }
+            }
+            const cleanAttr = {
+                idn: attr.idn,
+                value: processedValue,
+                title: attr.title || "",
+                description: attr.description || "",
+                group: attr.group || "",
+                is_hidden: attr.is_hidden,
+                possible_values: attr.possible_values || [],
+                value_type: `__ENUM_PLACEHOLDER_${attr.value_type}__`
+            };
+            return cleanAttr;
+        }) : [];
+        const attributesYaml = {
+            attributes: cleanAttributes
+        };
+        // Configure YAML output to match reference format exactly
+        let yamlContent = yaml.dump(attributesYaml, {
+            indent: 2,
+            quotingType: '"',
+            forceQuotes: false,
+            lineWidth: 80, // Wrap long lines to match reference format
+            noRefs: true,
+            sortKeys: false,
+            flowLevel: -1, // Never use flow syntax
+            styles: {
+                '!!str': 'folded' // Use folded style for better line wrapping of long strings
+            }
+        });
+        // Post-process to fix enum format and improve JSON string formatting
+        yamlContent = yamlContent.replace(/__ENUM_PLACEHOLDER_(\w+)__/g, '!enum "AttributeValueTypes.$1"');
+        // Fix JSON string formatting to match reference (remove escape characters)
+        yamlContent = yamlContent.replace(/\\"/g, '"');
+        // Save all files: attributes.yaml, ID mapping, and backup for diff tracking
+        await writeFileSafe(customerAttributesPath(customer.idn), yamlContent);
+        await writeFileSafe(customerAttributesMapPath(customer.idn), JSON.stringify(idMapping, null, 2));
+        await writeFileSafe(customerAttributesBackupPath(customer.idn), yamlContent);
+        if (verbose) {
+            console.log(`✓ Saved customer attributes to ${customerAttributesPath(customer.idn)}`);
+            console.log(`✓ Saved attribute ID mapping to ${customerAttributesMapPath(customer.idn)}`);
+            console.log(`✓ Created attributes backup for diff tracking`);
+        }
+        // Return content for hash calculation
+        return yamlContent;
+    }
+    catch (error) {
+        console.error(`❌ Failed to save customer attributes for ${customer.idn}:`, error);
+        throw error;
+    }
+}
+//# sourceMappingURL=attributes.js.map

package/dist/sync/conversations.d.ts

@@ -0,0 +1,7 @@
+import type { AxiosInstance } from 'axios';
+import type { CustomerConfig, ConversationOptions } from '../types.js';
+/**
+ * Pull conversations for a customer and save to YAML
+ */
+export declare function pullConversations(client: AxiosInstance, customer: CustomerConfig, options?: ConversationOptions, verbose?: boolean): Promise<void>;
+//# sourceMappingURL=conversations.d.ts.map

package/dist/sync/conversations.js

@@ -0,0 +1,218 @@
+/**
+ * Conversations synchronization module
+ */
+import { listUserPersonas, getChatHistory } from '../api.js';
+import { writeFileSafe } from '../fsutil.js';
+import yaml from 'js-yaml';
+import pLimit from 'p-limit';
+// Concurrency limit for API calls
+const concurrencyLimit = pLimit(5);
+/**
+ * Pull conversations for a customer and save to YAML
+ */
+export async function pullConversations(client, customer, options = {}, verbose = false) {
+    if (verbose)
+        console.log(`💬 Fetching conversations for customer ${customer.idn}...`);
+    try {
+        // Get all user personas with pagination
+        const allPersonas = [];
+        let page = 1;
+        const perPage = 50;
+        let hasMore = true;
+        while (hasMore) {
+            const response = await listUserPersonas(client, page, perPage);
+            allPersonas.push(...response.items);
+            if (verbose)
+                console.log(`📋 Page ${page}: Found ${response.items.length} personas (${allPersonas.length}/${response.metadata.total} total)`);
+            hasMore = response.items.length === perPage && allPersonas.length < response.metadata.total;
+            page++;
+        }
+        if (options.maxPersonas && allPersonas.length > options.maxPersonas) {
+            allPersonas.splice(options.maxPersonas);
+            if (verbose)
+                console.log(`⚠️  Limited to ${options.maxPersonas} personas as requested`);
+        }
+        if (verbose)
+            console.log(`👥 Processing ${allPersonas.length} personas...`);
+        // Process personas concurrently with limited concurrency
+        const processedPersonas = [];
+        await Promise.all(allPersonas.map(persona => concurrencyLimit(async () => {
+            try {
+                // Extract phone number from actors
+                const phoneActor = persona.actors.find(actor => actor.integration_idn === 'newo_voice' &&
+                    actor.connector_idn === 'newo_voice_connector' &&
+                    actor.contact_information?.startsWith('+'));
+                const phone = phoneActor?.contact_information || null;
+                // Get acts for this persona
+                const allActs = [];
+                let actPage = 1;
+                const actsPerPage = 100; // Higher limit for acts
+                let hasMoreActs = true;
+                // Get user actor IDs from persona actors first
+                const userActors = persona.actors.filter(actor => actor.integration_idn === 'newo_voice' &&
+                    actor.connector_idn === 'newo_voice_connector');
+                if (userActors.length === 0) {
+                    if (verbose)
+                        console.log(`  👤 ${persona.name}: No voice actors found, skipping`);
+                    // No voice actors, can't get chat history - add persona with empty acts
+                    processedPersonas.push({
+                        id: persona.id,
+                        name: persona.name,
+                        phone,
+                        act_count: persona.act_count,
+                        acts: []
+                    });
+                    if (verbose)
+                        console.log(`  ✓ Processed ${persona.name}: 0 acts (no voice actors)`);
+                    return; // Return from the concurrency function
+                }
+                // Safety mechanism to prevent infinite loops
+                const maxPages = 50; // Limit to 50 pages (5000 acts max per persona)
+                while (hasMoreActs && actPage <= maxPages) {
+                    try {
+                        const chatHistoryParams = {
+                            user_actor_id: userActors[0].id,
+                            page: actPage,
+                            per: actsPerPage
+                        };
+                        if (verbose)
+                            console.log(`    📄 ${persona.name}: Fetching page ${actPage}...`);
+                        const chatResponse = await getChatHistory(client, chatHistoryParams);
+                        if (chatResponse.items && chatResponse.items.length > 0) {
+                            // Convert chat history format to acts format - create minimal ConversationAct objects
+                            const convertedActs = chatResponse.items.map((item) => ({
+                                id: item.id || `chat_${Math.random()}`,
+                                command_act_id: null,
+                                external_event_id: item.external_event_id || 'chat_history',
+                                arguments: [],
+                                reference_idn: (item.is_agent === true) ? 'agent_message' : 'user_message',
+                                runtime_context_id: item.runtime_context_id || 'chat_history',
+                                source_text: item.payload?.text || item.message || item.content || item.text || '',
+                                original_text: item.payload?.text || item.message || item.content || item.text || '',
+                                datetime: item.datetime || item.created_at || item.timestamp || new Date().toISOString(),
+                                user_actor_id: userActors[0].id,
+                                agent_actor_id: null,
+                                user_persona_id: persona.id,
+                                user_persona_name: persona.name,
+                                agent_persona_id: item.agent_persona_id || 'unknown',
+                                external_id: item.external_id || null,
+                                integration_idn: 'newo_voice',
+                                connector_idn: 'newo_voice_connector',
+                                to_integration_idn: null,
+                                to_connector_idn: null,
+                                is_agent: Boolean(item.is_agent === true),
+                                project_idn: null,
+                                flow_idn: item.flow_idn || 'unknown',
+                                skill_idn: item.skill_idn || 'unknown',
+                                session_id: item.session_id || 'unknown',
+                                recordings: item.recordings || [],
+                                contact_information: item.contact_information || null
+                            }));
+                            allActs.push(...convertedActs);
+                            if (verbose && convertedActs.length > 0) {
+                                console.log(`  👤 ${persona.name}: Chat History - ${convertedActs.length} messages (${allActs.length} total)`);
+                            }
+                            // Check if we should continue paginating
+                            const hasMetadata = chatResponse.metadata?.total !== undefined;
+                            const currentTotal = chatResponse.metadata?.total || 0;
+                            hasMoreActs = chatResponse.items.length === actsPerPage &&
+                                hasMetadata &&
+                                allActs.length < currentTotal;
+                            actPage++;
+                            if (verbose)
+                                console.log(`    📊 ${persona.name}: Page ${actPage - 1} done, ${allActs.length}/${currentTotal} total acts`);
+                        }
+                        else {
+                            // No more items
+                            hasMoreActs = false;
+                            if (verbose)
+                                console.log(`    📊 ${persona.name}: No more chat history items`);
+                        }
+                    }
+                    catch (chatError) {
+                        if (verbose)
+                            console.log(`  ⚠️  Chat history failed for ${persona.name}: ${chatError instanceof Error ? chatError.message : String(chatError)}`);
+                        hasMoreActs = false;
+                    }
+                }
+                if (actPage > maxPages) {
+                    if (verbose)
+                        console.log(`  ⚠️  ${persona.name}: Reached max pages limit (${maxPages}), stopping pagination`);
+                }
+                // Sort acts by datetime ascending (chronological order)
+                allActs.sort((a, b) => new Date(a.datetime).getTime() - new Date(b.datetime).getTime());
+                // Process acts into simplified format - exclude redundant fields
+                const processedActs = allActs.map(act => {
+                    const processedAct = {
+                        datetime: act.datetime,
+                        type: act.reference_idn,
+                        message: act.source_text
+                    };
+                    // Only include non-redundant fields
+                    if (act.contact_information) {
+                        processedAct.contact_information = act.contact_information;
+                    }
+                    if (act.flow_idn && act.flow_idn !== 'unknown') {
+                        processedAct.flow_idn = act.flow_idn;
+                    }
+                    if (act.skill_idn && act.skill_idn !== 'unknown') {
+                        processedAct.skill_idn = act.skill_idn;
+                    }
+                    if (act.session_id && act.session_id !== 'unknown') {
+                        processedAct.session_id = act.session_id;
+                    }
+                    return processedAct;
+                });
+                processedPersonas.push({
+                    id: persona.id,
+                    name: persona.name,
+                    phone,
+                    act_count: persona.act_count,
+                    acts: processedActs
+                });
+                if (verbose)
+                    console.log(`  ✓ Processed ${persona.name}: ${processedActs.length} acts`);
+            }
+            catch (error) {
+                console.error(`❌ Failed to process persona ${persona.name}:`, error);
+                // Continue with other personas
+            }
+        })));
+        // Sort personas by most recent act time (descending) - use latest act from acts array
+        processedPersonas.sort((a, b) => {
+            const aLatestTime = a.acts.length > 0 ? a.acts[a.acts.length - 1].datetime : '1970-01-01T00:00:00.000Z';
+            const bLatestTime = b.acts.length > 0 ? b.acts[b.acts.length - 1].datetime : '1970-01-01T00:00:00.000Z';
+            return new Date(bLatestTime).getTime() - new Date(aLatestTime).getTime();
+        });
+        // Calculate totals
+        const totalActs = processedPersonas.reduce((sum, persona) => sum + persona.acts.length, 0);
+        // Create final conversations data
+        const conversationsData = {
+            personas: processedPersonas,
+            total_personas: processedPersonas.length,
+            total_acts: totalActs,
+            generated_at: new Date().toISOString()
+        };
+        // Save to YAML file
+        const conversationsPath = `newo_customers/${customer.idn}/conversations.yaml`;
+        const yamlContent = yaml.dump(conversationsData, {
+            indent: 2,
+            quotingType: '"',
+            forceQuotes: false,
+            lineWidth: 120,
+            noRefs: true,
+            sortKeys: false,
+            flowLevel: -1
+        });
+        await writeFileSafe(conversationsPath, yamlContent);
+        if (verbose) {
+            console.log(`✓ Saved conversations to ${conversationsPath}`);
+            console.log(`📊 Summary: ${processedPersonas.length} personas, ${totalActs} total acts`);
+        }
+    }
+    catch (error) {
+        console.error(`❌ Failed to pull conversations for ${customer.idn}:`, error);
+        throw error;
+    }
+}
+//# sourceMappingURL=conversations.js.map

package/dist/sync/metadata.d.ts

@@ -0,0 +1,8 @@
+import type { ProjectData, ProjectMap } from '../types.js';
+/**
+ * Generate flows.yaml file from project data
+ */
+export declare function generateFlowsYaml(projectMap: ProjectMap | {
+    [key: string]: ProjectData;
+}, customerIdn: string, verbose?: boolean): Promise<string>;
+//# sourceMappingURL=metadata.d.ts.map

package/dist/sync/metadata.js

@@ -0,0 +1,124 @@
+/**
+ * Metadata and flows.yaml generation operations
+ */
+import { writeFileSafe, flowMetadataPath, agentMetadataPath, flowsYamlPath } from '../fsutil.js';
+import fs from 'fs-extra';
+import yaml from 'js-yaml';
+/**
+ * Generate flows.yaml file from project data
+ */
+export async function generateFlowsYaml(projectMap, customerIdn, verbose = false) {
+    if (verbose)
+        console.log(`📊 Generating flows.yaml for customer ${customerIdn}...`);
+    const flowsData = {
+        flows: []
+    };
+    // Handle both formats
+    const projects = 'projects' in projectMap ? projectMap.projects : projectMap;
+    for (const [projectIdn, projectData] of Object.entries(projects)) {
+        if (verbose && projectIdn)
+            console.log(`  📁 Processing project: ${projectIdn}`);
+        for (const [agentIdn, agentData] of Object.entries(projectData.agents)) {
+            if (verbose)
+                console.log(`    📁 Processing agent: ${agentIdn}`);
+            const agentFlows = [];
+            for (const [flowIdn, flowData] of Object.entries(agentData.flows)) {
+                if (verbose)
+                    console.log(`      📁 Processing flow: ${flowIdn}`);
+                // Load flow metadata to get comprehensive flow information
+                const flowMetaPath = flowMetadataPath(customerIdn, projectIdn, agentIdn, flowIdn);
+                let flowMeta = null;
+                try {
+                    if (await fs.pathExists(flowMetaPath)) {
+                        const flowMetaContent = await fs.readFile(flowMetaPath, 'utf8');
+                        flowMeta = yaml.load(flowMetaContent);
+                    }
+                }
+                catch (e) {
+                    if (verbose)
+                        console.log(`        ⚠️  Could not load flow metadata: ${flowMetaPath}`);
+                }
+                const skills = [];
+                for (const [, skillMeta] of Object.entries(flowData.skills)) {
+                    // Note: We don't need to load script content since prompt_script is excluded from flows.yaml
+                    skills.push({
+                        idn: skillMeta.idn,
+                        title: skillMeta.title,
+                        runner_type: skillMeta.runner_type,
+                        model: skillMeta.model,
+                        parameters: skillMeta.parameters.map((p) => ({
+                            name: p.name,
+                            default_value: p.default_value || ''
+                        }))
+                    });
+                }
+                // Use flow metadata if available, otherwise use basic info
+                const flowYaml = {
+                    idn: flowIdn,
+                    title: flowMeta?.title || 'Unknown Flow',
+                    description: flowMeta?.description || null,
+                    default_runner_type: flowMeta?.default_runner_type || 'guidance',
+                    default_provider_idn: flowMeta?.default_model?.provider_idn || 'openai',
+                    default_model_idn: flowMeta?.default_model?.model_idn || 'gpt-4',
+                    skills,
+                    events: flowMeta?.events?.map(event => ({
+                        title: event.description,
+                        idn: event.idn,
+                        skill_selector: event.skill_selector,
+                        skill_idn: event.skill_idn || null,
+                        state_idn: event.state_idn || null,
+                        integration_idn: event.integration_idn || null,
+                        connector_idn: event.connector_idn || null,
+                        interrupt_mode: event.interrupt_mode
+                    })) || [],
+                    state_fields: flowMeta?.state_fields?.map(state => ({
+                        title: state.title,
+                        idn: state.idn,
+                        default_value: state.default_value || null,
+                        scope: state.scope
+                    })) || []
+                };
+                agentFlows.push(flowYaml);
+            }
+            if (agentFlows.length > 0) {
+                // Load agent metadata for description
+                const agentMetaPath = agentMetadataPath(customerIdn, projectIdn, agentIdn);
+                let agentDescription = null;
+                try {
+                    if (await fs.pathExists(agentMetaPath)) {
+                        const agentMetaContent = await fs.readFile(agentMetaPath, 'utf8');
+                        const agentMeta = yaml.load(agentMetaContent);
+                        agentDescription = agentMeta.description || null;
+                    }
+                }
+                catch (e) {
+                    if (verbose)
+                        console.log(`      ⚠️  Could not load agent metadata: ${agentMetaPath}`);
+                }
+                flowsData.flows.push({
+                    agent_idn: agentIdn,
+                    agent_description: agentDescription,
+                    agent_flows: agentFlows
+                });
+            }
+        }
+    }
+    // Generate flows.yaml content
+    const flowsYamlContent = yaml.dump(flowsData, {
+        indent: 2,
+        quotingType: '"',
+        forceQuotes: false,
+        lineWidth: 120,
+        noRefs: true,
+        sortKeys: false,
+        flowLevel: -1
+    });
+    // Save flows.yaml
+    const flowsFilePath = flowsYamlPath(customerIdn);
+    await writeFileSafe(flowsFilePath, flowsYamlContent);
+    if (verbose)
+        console.log(`✓ Generated flows.yaml with ${flowsData.flows.length} agents`);
+    // Return content for hash calculation
+    return flowsYamlContent;
+}
+//# sourceMappingURL=metadata.js.map

package/dist/sync/projects.d.ts

@@ -0,0 +1,13 @@
+import type { AxiosInstance } from 'axios';
+import type { ProjectMap, LegacyProjectMap, CustomerConfig } from '../types.js';
+export declare function isProjectMap(x: unknown): x is ProjectMap;
+export declare function isLegacyProjectMap(x: unknown): x is LegacyProjectMap;
+/**
+ * Pull a single project and all its data
+ */
+export declare function pullSingleProject(client: AxiosInstance, customer: CustomerConfig, projectId: string | null, verbose?: boolean, silentOverwrite?: boolean): Promise<void>;
+/**
+ * Pull all projects for a customer
+ */
+export declare function pullAll(client: AxiosInstance, customer: CustomerConfig, projectId?: string | null, verbose?: boolean, silentOverwrite?: boolean): Promise<void>;
+//# sourceMappingURL=projects.d.ts.map