newo 1.9.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +116 -0
  2. package/README.md +68 -20
  3. package/dist/cli/commands/conversations.d.ts +3 -0
  4. package/dist/cli/commands/conversations.js +38 -0
  5. package/dist/cli/commands/help.d.ts +5 -0
  6. package/dist/cli/commands/help.js +50 -0
  7. package/dist/cli/commands/import-akb.d.ts +3 -0
  8. package/dist/cli/commands/import-akb.js +62 -0
  9. package/dist/cli/commands/list-customers.d.ts +3 -0
  10. package/dist/cli/commands/list-customers.js +13 -0
  11. package/dist/cli/commands/meta.d.ts +3 -0
  12. package/dist/cli/commands/meta.js +19 -0
  13. package/dist/cli/commands/pull-attributes.d.ts +3 -0
  14. package/dist/cli/commands/pull-attributes.js +16 -0
  15. package/dist/cli/commands/pull.d.ts +3 -0
  16. package/dist/cli/commands/pull.js +34 -0
  17. package/dist/cli/commands/push.d.ts +3 -0
  18. package/dist/cli/commands/push.js +39 -0
  19. package/dist/cli/commands/status.d.ts +3 -0
  20. package/dist/cli/commands/status.js +22 -0
  21. package/dist/cli/customer-selection.d.ts +23 -0
  22. package/dist/cli/customer-selection.js +110 -0
  23. package/dist/cli/errors.d.ts +9 -0
  24. package/dist/cli/errors.js +111 -0
  25. package/dist/cli.js +66 -463
  26. package/dist/fsutil.js +1 -1
  27. package/dist/sync/attributes.d.ts +7 -0
  28. package/dist/sync/attributes.js +90 -0
  29. package/dist/sync/conversations.d.ts +7 -0
  30. package/dist/sync/conversations.js +218 -0
  31. package/dist/sync/metadata.d.ts +8 -0
  32. package/dist/sync/metadata.js +124 -0
  33. package/dist/sync/projects.d.ts +13 -0
  34. package/dist/sync/projects.js +283 -0
  35. package/dist/sync/push.d.ts +7 -0
  36. package/dist/sync/push.js +171 -0
  37. package/dist/sync/skill-files.d.ts +42 -0
  38. package/dist/sync/skill-files.js +121 -0
  39. package/dist/sync/status.d.ts +6 -0
  40. package/dist/sync/status.js +247 -0
  41. package/dist/sync.d.ts +10 -8
  42. package/dist/sync.js +12 -1226
  43. package/dist/types.d.ts +0 -1
  44. package/package.json +2 -2
  45. package/src/cli/commands/conversations.ts +47 -0
  46. package/src/cli/commands/help.ts +50 -0
  47. package/src/cli/commands/import-akb.ts +71 -0
  48. package/src/cli/commands/list-customers.ts +14 -0
  49. package/src/cli/commands/meta.ts +26 -0
  50. package/src/cli/commands/pull-attributes.ts +23 -0
  51. package/src/cli/commands/pull.ts +43 -0
  52. package/src/cli/commands/push.ts +47 -0
  53. package/src/cli/commands/status.ts +30 -0
  54. package/src/cli/customer-selection.ts +135 -0
  55. package/src/cli/errors.ts +111 -0
  56. package/src/cli.ts +77 -471
  57. package/src/fsutil.ts +1 -1
  58. package/src/sync/attributes.ts +110 -0
  59. package/src/sync/conversations.ts +257 -0
  60. package/src/sync/metadata.ts +153 -0
  61. package/src/sync/projects.ts +359 -0
  62. package/src/sync/push.ts +200 -0
  63. package/src/sync/skill-files.ts +176 -0
  64. package/src/sync/status.ts +277 -0
  65. package/src/sync.ts +14 -1418
  66. package/src/types.ts +0 -1
package/dist/sync.js CHANGED
@@ -1,1227 +1,13 @@
1
- import { listProjects, listAgents, listFlowSkills, updateSkill, listFlowEvents, listFlowStates, getProjectMeta, getCustomerAttributes, updateCustomerAttribute, listUserPersonas, getChatHistory } from './api.js';
2
- import { ensureState, skillPath, skillScriptPath, writeFileSafe, readIfExists, mapPath, projectMetadataPath, agentMetadataPath, flowMetadataPath, skillMetadataPath, flowsYamlPath, customerAttributesPath, customerAttributesMapPath, customerAttributesBackupPath } from './fsutil.js';
3
- import fs from 'fs-extra';
4
- import { sha256, loadHashes, saveHashes } from './hash.js';
5
- import yaml from 'js-yaml';
6
- import pLimit from 'p-limit';
7
- // Concurrency limits for API operations
8
- const concurrencyLimit = pLimit(5);
9
- // Type guards for better type safety
10
- function isProjectMap(x) {
11
- return !!x && typeof x === 'object' && 'projects' in x;
12
- }
13
- function isLegacyProjectMap(x) {
14
- return !!x && typeof x === 'object' && 'agents' in x;
15
- }
16
- export async function saveCustomerAttributes(client, customer, verbose = false) {
17
- if (verbose)
18
- console.log(`🔍 Fetching customer attributes for ${customer.idn}...`);
19
- try {
20
- const response = await getCustomerAttributes(client, true); // Include hidden attributes
21
- // API returns { groups: [...], attributes: [...] }
22
- // We only want the attributes array in the expected format
23
- const attributes = response.attributes || response;
24
- if (verbose)
25
- console.log(`📦 Found ${Array.isArray(attributes) ? attributes.length : 'invalid'} attributes`);
26
- // Create ID mapping for push operations (separate from YAML)
27
- const idMapping = {};
28
- // Transform attributes to match reference format exactly (no ID fields)
29
- const cleanAttributes = Array.isArray(attributes) ? attributes.map(attr => {
30
- // Store ID mapping for push operations
31
- if (attr.id) {
32
- idMapping[attr.idn] = attr.id;
33
- }
34
- // Special handling for complex JSON string values
35
- let processedValue = attr.value;
36
- if (typeof attr.value === 'string' && attr.value.startsWith('[{') && attr.value.endsWith('}]')) {
37
- try {
38
- // Parse and reformat JSON for better readability
39
- const parsed = JSON.parse(attr.value);
40
- processedValue = JSON.stringify(parsed, null, 0); // No extra spacing, but valid JSON
41
- }
42
- catch (e) {
43
- // Keep original if parsing fails
44
- processedValue = attr.value;
45
- }
46
- }
47
- const cleanAttr = {
48
- idn: attr.idn,
49
- value: processedValue,
50
- title: attr.title || "",
51
- description: attr.description || "",
52
- group: attr.group || "",
53
- is_hidden: attr.is_hidden,
54
- possible_values: attr.possible_values || [],
55
- value_type: `__ENUM_PLACEHOLDER_${attr.value_type}__`
56
- };
57
- return cleanAttr;
58
- }) : [];
59
- const attributesYaml = {
60
- attributes: cleanAttributes
61
- };
62
- // Configure YAML output to match reference format exactly
63
- let yamlContent = yaml.dump(attributesYaml, {
64
- indent: 2,
65
- quotingType: '"',
66
- forceQuotes: false,
67
- lineWidth: 80, // Wrap long lines to match reference format
68
- noRefs: true,
69
- sortKeys: false,
70
- flowLevel: -1, // Never use flow syntax
71
- styles: {
72
- '!!str': 'folded' // Use folded style for better line wrapping of long strings
73
- }
74
- });
75
- // Post-process to fix enum format and improve JSON string formatting
76
- yamlContent = yamlContent.replace(/__ENUM_PLACEHOLDER_(\w+)__/g, '!enum "AttributeValueTypes.$1"');
77
- // Fix JSON string formatting to match reference (remove escape characters)
78
- yamlContent = yamlContent.replace(/\\"/g, '"');
79
- // Save all files: attributes.yaml, ID mapping, and backup for diff tracking
80
- await writeFileSafe(customerAttributesPath(customer.idn), yamlContent);
81
- await writeFileSafe(customerAttributesMapPath(customer.idn), JSON.stringify(idMapping, null, 2));
82
- await writeFileSafe(customerAttributesBackupPath(customer.idn), yamlContent);
83
- if (verbose) {
84
- console.log(`✓ Saved customer attributes to ${customerAttributesPath(customer.idn)}`);
85
- console.log(`✓ Saved attribute ID mapping to ${customerAttributesMapPath(customer.idn)}`);
86
- console.log(`✓ Created attributes backup for diff tracking`);
87
- }
88
- }
89
- catch (error) {
90
- console.error(`❌ Failed to save customer attributes for ${customer.idn}:`, error);
91
- throw error;
92
- }
93
- }
94
- export async function pullSingleProject(client, customer, projectId, projectIdn, verbose = false) {
95
- if (verbose)
96
- console.log(`🔍 Fetching agents for project ${projectId} (${projectIdn}) for customer ${customer.idn}...`);
97
- const agents = await listAgents(client, projectId);
98
- if (verbose)
99
- console.log(`📦 Found ${agents.length} agents`);
100
- // Get and create project metadata
101
- const projectMeta = await getProjectMeta(client, projectId);
102
- const projectMetadata = {
103
- id: projectMeta.id,
104
- idn: projectMeta.idn,
105
- title: projectMeta.title,
106
- ...(projectMeta.description && { description: projectMeta.description }),
107
- ...(projectMeta.created_at && { created_at: projectMeta.created_at }),
108
- ...(projectMeta.updated_at && { updated_at: projectMeta.updated_at })
109
- };
110
- await writeFileSafe(projectMetadataPath(customer.idn, projectIdn), yaml.dump(projectMetadata, { indent: 2 }));
111
- if (verbose)
112
- console.log(`✓ Created project metadata.yaml for ${projectIdn}`);
113
- // Legacy metadata.json generation removed - YAML is sufficient
114
- const projectMap = { projectId, projectIdn, agents: {} };
115
- for (const agent of agents) {
116
- const aKey = agent.idn;
117
- projectMap.agents[aKey] = { id: agent.id, flows: {} };
118
- // Create agent metadata
119
- const agentMetadata = {
120
- id: agent.id,
121
- idn: agent.idn,
122
- ...(agent.title && { title: agent.title }),
123
- ...(agent.description && { description: agent.description })
124
- };
125
- await writeFileSafe(agentMetadataPath(customer.idn, projectIdn, agent.idn), yaml.dump(agentMetadata, { indent: 2 }));
126
- if (verbose)
127
- console.log(` ✓ Created agent metadata for ${agent.idn}`);
128
- for (const flow of agent.flows ?? []) {
129
- projectMap.agents[aKey].flows[flow.idn] = { id: flow.id, skills: {} };
130
- // Fetch flow events and state fields for metadata
131
- let flowEvents = [];
132
- let flowStates = [];
133
- try {
134
- flowEvents = await listFlowEvents(client, flow.id);
135
- if (verbose)
136
- console.log(` 📋 Found ${flowEvents.length} events for flow ${flow.idn}`);
137
- }
138
- catch (error) {
139
- if (verbose)
140
- console.log(` ⚠️ No events found for flow ${flow.idn}`);
141
- }
142
- try {
143
- flowStates = await listFlowStates(client, flow.id);
144
- if (verbose)
145
- console.log(` 📊 Found ${flowStates.length} state fields for flow ${flow.idn}`);
146
- }
147
- catch (error) {
148
- if (verbose)
149
- console.log(` ⚠️ No state fields found for flow ${flow.idn}`);
150
- }
151
- // Create flow metadata
152
- const flowMetadata = {
153
- id: flow.id,
154
- idn: flow.idn,
155
- title: flow.title,
156
- ...(flow.description && { description: flow.description }),
157
- default_runner_type: flow.default_runner_type,
158
- default_model: flow.default_model,
159
- events: flowEvents,
160
- state_fields: flowStates
161
- };
162
- await writeFileSafe(flowMetadataPath(customer.idn, projectIdn, agent.idn, flow.idn), yaml.dump(flowMetadata, { indent: 2 }));
163
- if (verbose)
164
- console.log(` ✓ Created flow metadata for ${flow.idn}`);
165
- const skills = await listFlowSkills(client, flow.id);
166
- // Process skills concurrently with limited concurrency
167
- await Promise.all(skills.map(skill => concurrencyLimit(async () => {
168
- // Create skill folder and script file
169
- const scriptFile = skillScriptPath(customer.idn, projectIdn, agent.idn, flow.idn, skill.idn, skill.runner_type);
170
- await writeFileSafe(scriptFile, skill.prompt_script || '');
171
- // Create skill metadata
172
- const skillMetadata = {
173
- id: skill.id,
174
- idn: skill.idn,
175
- title: skill.title,
176
- runner_type: skill.runner_type,
177
- model: skill.model,
178
- parameters: [...skill.parameters],
179
- path: skill.path || undefined
180
- };
181
- const skillMetaFile = skillMetadataPath(customer.idn, projectIdn, agent.idn, flow.idn, skill.idn);
182
- await writeFileSafe(skillMetaFile, yaml.dump(skillMetadata, { indent: 2 }));
183
- // Store complete skill metadata for push operations (keep for backwards compatibility)
184
- projectMap.agents[aKey].flows[flow.idn].skills[skill.idn] = {
185
- id: skill.id,
186
- title: skill.title,
187
- idn: skill.idn,
188
- runner_type: skill.runner_type,
189
- model: skill.model,
190
- parameters: [...skill.parameters],
191
- path: skill.path || undefined
192
- };
193
- console.log(`✓ Created skill folder and metadata for ${skill.idn}`);
194
- })));
195
- }
196
- }
197
- // Generate flows.yaml for this project (backwards compatibility)
198
- if (verbose)
199
- console.log(`📄 Generating flows.yaml...`);
200
- await generateFlowsYaml(client, customer, agents, verbose);
201
- return projectMap;
202
- }
203
- export async function pullAll(client, customer, projectId = null, verbose = false) {
204
- await ensureState(customer.idn);
205
- if (projectId) {
206
- // Single project mode
207
- const projectMeta = await getProjectMeta(client, projectId);
208
- const projectMap = await pullSingleProject(client, customer, projectId, projectMeta.idn, verbose);
209
- const idMap = { projects: { [projectMeta.idn]: projectMap } };
210
- await fs.writeJson(mapPath(customer.idn), idMap, { spaces: 2 });
211
- // Generate hash tracking for this project (both legacy and new paths)
212
- const hashes = {};
213
- for (const [agentIdn, agentObj] of Object.entries(projectMap.agents)) {
214
- for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
215
- for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
216
- // Track new skill script path
217
- const newPath = skillScriptPath(customer.idn, projectMeta.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
218
- const content = await fs.readFile(newPath, 'utf8');
219
- hashes[newPath] = sha256(content);
220
- // Track skill metadata.yaml file
221
- const metadataPath = skillMetadataPath(customer.idn, projectMeta.idn, agentIdn, flowIdn, skillIdn);
222
- if (await fs.pathExists(metadataPath)) {
223
- const metadataContent = await fs.readFile(metadataPath, 'utf8');
224
- hashes[metadataPath] = sha256(metadataContent);
225
- }
226
- // Also track legacy path for backwards compatibility during transition
227
- const legacyPath = skillPath(customer.idn, projectMeta.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
228
- hashes[legacyPath] = sha256(content);
229
- }
230
- }
231
- }
232
- // Save customer attributes before hash tracking
233
- try {
234
- await saveCustomerAttributes(client, customer, verbose);
235
- // Add attributes.yaml to hash tracking
236
- const attributesFile = customerAttributesPath(customer.idn);
237
- if (await fs.pathExists(attributesFile)) {
238
- const attributesContent = await fs.readFile(attributesFile, 'utf8');
239
- hashes[attributesFile] = sha256(attributesContent);
240
- if (verbose)
241
- console.log(`✓ Added attributes.yaml to hash tracking`);
242
- }
243
- // Add flows.yaml to hash tracking
244
- const flowsFile = flowsYamlPath(customer.idn);
245
- if (await fs.pathExists(flowsFile)) {
246
- const flowsContent = await fs.readFile(flowsFile, 'utf8');
247
- hashes[flowsFile] = sha256(flowsContent);
248
- if (verbose)
249
- console.log(`✓ Added flows.yaml to hash tracking`);
250
- }
251
- }
252
- catch (error) {
253
- console.error(`❌ Failed to save customer attributes for ${customer.idn}:`, error);
254
- // Don't throw - continue with the rest of the process
255
- }
256
- await saveHashes(hashes, customer.idn);
257
- return;
258
- }
259
- // Multi-project mode
260
- if (verbose)
261
- console.log(`🔍 Fetching all projects for customer ${customer.idn}...`);
262
- const projects = await listProjects(client);
263
- if (verbose)
264
- console.log(`📦 Found ${projects.length} projects`);
265
- const idMap = { projects: {} };
266
- const allHashes = {};
267
- for (const project of projects) {
268
- if (verbose)
269
- console.log(`\n📁 Processing project: ${project.idn} (${project.title})`);
270
- const projectMap = await pullSingleProject(client, customer, project.id, project.idn, verbose);
271
- idMap.projects[project.idn] = projectMap;
272
- // Collect hashes for this project (both legacy and new paths)
273
- for (const [agentIdn, agentObj] of Object.entries(projectMap.agents)) {
274
- for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
275
- for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
276
- // Track new skill script path
277
- const newPath = skillScriptPath(customer.idn, project.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
278
- const content = await fs.readFile(newPath, 'utf8');
279
- allHashes[newPath] = sha256(content);
280
- // Track skill metadata.yaml file
281
- const metadataPath = skillMetadataPath(customer.idn, project.idn, agentIdn, flowIdn, skillIdn);
282
- if (await fs.pathExists(metadataPath)) {
283
- const metadataContent = await fs.readFile(metadataPath, 'utf8');
284
- allHashes[metadataPath] = sha256(metadataContent);
285
- }
286
- // Also track legacy path for backwards compatibility during transition
287
- const legacyPath = skillPath(customer.idn, project.idn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
288
- allHashes[legacyPath] = sha256(content);
289
- }
290
- }
291
- }
292
- }
293
- await fs.writeJson(mapPath(customer.idn), idMap, { spaces: 2 });
294
- // Save customer attributes before hash tracking
295
- try {
296
- await saveCustomerAttributes(client, customer, verbose);
297
- // Add attributes.yaml to hash tracking
298
- const attributesFile = customerAttributesPath(customer.idn);
299
- if (await fs.pathExists(attributesFile)) {
300
- const attributesContent = await fs.readFile(attributesFile, 'utf8');
301
- allHashes[attributesFile] = sha256(attributesContent);
302
- if (verbose)
303
- console.log(`✓ Added attributes.yaml to hash tracking`);
304
- }
305
- // Add flows.yaml to hash tracking
306
- const flowsFile = flowsYamlPath(customer.idn);
307
- if (await fs.pathExists(flowsFile)) {
308
- const flowsContent = await fs.readFile(flowsFile, 'utf8');
309
- allHashes[flowsFile] = sha256(flowsContent);
310
- if (verbose)
311
- console.log(`✓ Added flows.yaml to hash tracking`);
312
- }
313
- }
314
- catch (error) {
315
- console.error(`❌ Failed to save customer attributes for ${customer.idn}:`, error);
316
- // Don't throw - continue with the rest of the process
317
- }
318
- await saveHashes(allHashes, customer.idn);
319
- }
320
- export async function pushChanged(client, customer, verbose = false) {
321
- await ensureState(customer.idn);
322
- if (!(await fs.pathExists(mapPath(customer.idn)))) {
323
- throw new Error(`Missing .newo/${customer.idn}/map.json. Run \`newo pull --customer ${customer.idn}\` first.`);
324
- }
325
- if (verbose)
326
- console.log(`📋 Loading project mapping for customer ${customer.idn}...`);
327
- const idMapData = await fs.readJson(mapPath(customer.idn));
328
- if (verbose)
329
- console.log('🔍 Loading file hashes...');
330
- const oldHashes = await loadHashes(customer.idn);
331
- const newHashes = { ...oldHashes };
332
- if (verbose)
333
- console.log('🔄 Scanning for changes...');
334
- let pushed = 0;
335
- let scanned = 0;
336
- let metadataChanged = false;
337
- // Handle both old single-project format and new multi-project format with type guards
338
- const projects = isProjectMap(idMapData) && idMapData.projects
339
- ? idMapData.projects
340
- : isLegacyProjectMap(idMapData)
341
- ? { '': idMapData }
342
- : (() => { throw new Error('Invalid project map format'); })();
343
- for (const [projectIdn, projectData] of Object.entries(projects)) {
344
- if (verbose && projectIdn)
345
- console.log(`📁 Scanning project: ${projectIdn}`);
346
- for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
347
- if (verbose)
348
- console.log(` 📁 Scanning agent: ${agentIdn}`);
349
- for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
350
- if (verbose)
351
- console.log(` 📁 Scanning flow: ${flowIdn}`);
352
- for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
353
- // Try new folder structure first
354
- const newPath = projectIdn ?
355
- skillScriptPath(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
356
- skillScriptPath(customer.idn, '', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
357
- // Fallback to legacy structure
358
- const legacyPath = projectIdn ?
359
- skillPath(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
360
- skillPath(customer.idn, '', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
361
- let currentPath = newPath;
362
- let content = await readIfExists(newPath);
363
- // If new structure doesn't exist, try legacy structure
364
- if (content === null) {
365
- content = await readIfExists(legacyPath);
366
- currentPath = legacyPath;
367
- }
368
- scanned++;
369
- if (verbose)
370
- console.log(` 📄 Checking: ${currentPath}`);
371
- if (content === null) {
372
- if (verbose)
373
- console.log(` ⚠️ File not found: ${currentPath}`);
374
- continue;
375
- }
376
- const h = sha256(content);
377
- const oldHash = oldHashes[currentPath];
378
- if (verbose) {
379
- console.log(` 🔍 Hash comparison:`);
380
- console.log(` Old: ${oldHash || 'none'}`);
381
- console.log(` New: ${h}`);
382
- }
383
- if (oldHash !== h) {
384
- if (verbose)
385
- console.log(` 🔄 File changed, preparing to push...`);
386
- // For new folder structure, try to load metadata from YAML file
387
- let skillMetadata = skillMeta;
388
- if (currentPath === newPath) {
389
- const metadataFile = projectIdn ?
390
- skillMetadataPath(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn) :
391
- skillMetadataPath(customer.idn, '', agentIdn, flowIdn, skillIdn);
392
- const metadataContent = await readIfExists(metadataFile);
393
- if (metadataContent) {
394
- try {
395
- const yamlMetadata = yaml.load(metadataContent);
396
- skillMetadata = yamlMetadata;
397
- if (verbose)
398
- console.log(` 📄 Loaded skill metadata from ${metadataFile}`);
399
- }
400
- catch (error) {
401
- if (verbose)
402
- console.log(` ⚠️ Failed to parse skill metadata, using project map data`);
403
- }
404
- }
405
- }
406
- // Create complete skill object with updated prompt_script
407
- const skillObject = {
408
- id: skillMetadata.id,
409
- title: skillMetadata.title,
410
- idn: skillMetadata.idn,
411
- prompt_script: content,
412
- runner_type: skillMetadata.runner_type,
413
- model: skillMetadata.model,
414
- parameters: skillMetadata.parameters,
415
- path: skillMetadata.path || undefined
416
- };
417
- if (verbose) {
418
- console.log(` 📤 Pushing skill object:`);
419
- console.log(` ID: ${skillObject.id}`);
420
- console.log(` Title: ${skillObject.title}`);
421
- console.log(` IDN: ${skillObject.idn}`);
422
- console.log(` Content length: ${content.length} chars`);
423
- console.log(` Content preview: ${content.substring(0, 100).replace(/\n/g, '\\n')}...`);
424
- }
425
- await updateSkill(client, skillObject);
426
- console.log(`↑ Pushed ${currentPath}`);
427
- newHashes[currentPath] = h;
428
- pushed++;
429
- }
430
- else if (verbose) {
431
- console.log(` ✓ No changes`);
432
- }
433
- }
434
- }
435
- }
436
- }
437
- // Check for metadata-only changes (when metadata changed but script didn't)
438
- try {
439
- for (const [projectIdn, projectData] of Object.entries(projects)) {
440
- for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
441
- for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
442
- for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
443
- const metadataPath = projectIdn ?
444
- skillMetadataPath(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn) :
445
- skillMetadataPath(customer.idn, '', agentIdn, flowIdn, skillIdn);
446
- if (await fs.pathExists(metadataPath)) {
447
- const metadataContent = await fs.readFile(metadataPath, 'utf8');
448
- const h = sha256(metadataContent);
449
- const oldHash = oldHashes[metadataPath];
450
- if (oldHash !== h) {
451
- if (verbose)
452
- console.log(`🔄 Metadata-only change detected for ${skillIdn}, updating skill...`);
453
- try {
454
- // Load updated metadata
455
- const updatedMetadata = yaml.load(metadataContent);
456
- // Get current script content
457
- const scriptPath = projectIdn ?
458
- skillScriptPath(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
459
- skillScriptPath(customer.idn, '', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
460
- let scriptContent = '';
461
- if (await fs.pathExists(scriptPath)) {
462
- scriptContent = await fs.readFile(scriptPath, 'utf8');
463
- }
464
- // Create skill object with updated metadata
465
- const skillObject = {
466
- id: updatedMetadata.id,
467
- title: updatedMetadata.title,
468
- idn: updatedMetadata.idn,
469
- prompt_script: scriptContent,
470
- runner_type: updatedMetadata.runner_type,
471
- model: updatedMetadata.model,
472
- parameters: updatedMetadata.parameters,
473
- path: updatedMetadata.path || undefined
474
- };
475
- await updateSkill(client, skillObject);
476
- console.log(`↑ Pushed metadata update for skill: ${skillIdn} (${updatedMetadata.title})`);
477
- newHashes[metadataPath] = h;
478
- pushed++;
479
- metadataChanged = true;
480
- }
481
- catch (error) {
482
- console.error(`❌ Failed to push metadata for ${skillIdn}: ${error instanceof Error ? error.message : String(error)}`);
483
- }
484
- }
485
- }
486
- }
487
- }
488
- }
489
- }
490
- }
491
- catch (error) {
492
- if (verbose)
493
- console.log(`⚠️ Metadata push check failed: ${error instanceof Error ? error.message : String(error)}`);
494
- }
495
- if (verbose)
496
- console.log(`🔄 Scanned ${scanned} files, found ${pushed} changes`);
497
- // Check for attributes changes and push specific changed attributes only
498
- try {
499
- const attributesFile = customerAttributesPath(customer.idn);
500
- const attributesMapFile = customerAttributesMapPath(customer.idn);
501
- const attributesBackupFile = customerAttributesBackupPath(customer.idn);
502
- if (await fs.pathExists(attributesFile) && await fs.pathExists(attributesMapFile)) {
503
- if (verbose)
504
- console.log('🔍 Checking customer attributes for changes...');
505
- const currentContent = await fs.readFile(attributesFile, 'utf8');
506
- // Check if backup exists for diff comparison
507
- if (await fs.pathExists(attributesBackupFile)) {
508
- const backupContent = await fs.readFile(attributesBackupFile, 'utf8');
509
- if (currentContent !== backupContent) {
510
- if (verbose)
511
- console.log(`🔄 Attributes file changed, analyzing differences...`);
512
- try {
513
- // Load ID mapping for push operations
514
- const idMapping = await fs.readJson(attributesMapFile);
515
- // Parse both versions to find changed attributes
516
- const parseYaml = (content) => {
517
- let yamlContent = content.replace(/!enum "([^"]+)"/g, '"$1"');
518
- return yaml.load(yamlContent);
519
- };
520
- const currentData = parseYaml(currentContent);
521
- const backupData = parseYaml(backupContent);
522
- if (currentData?.attributes && backupData?.attributes) {
523
- // Create maps for comparison
524
- const currentAttrs = new Map(currentData.attributes.map(attr => [attr.idn, attr]));
525
- const backupAttrs = new Map(backupData.attributes.map(attr => [attr.idn, attr]));
526
- let attributesPushed = 0;
527
- // Find changed attributes
528
- for (const [idn, currentAttr] of currentAttrs) {
529
- const backupAttr = backupAttrs.get(idn);
530
- // Check if attribute changed (deep comparison of key fields)
531
- const hasChanged = !backupAttr ||
532
- currentAttr.value !== backupAttr.value ||
533
- currentAttr.title !== backupAttr.title ||
534
- currentAttr.description !== backupAttr.description ||
535
- currentAttr.group !== backupAttr.group ||
536
- currentAttr.is_hidden !== backupAttr.is_hidden;
537
- if (hasChanged) {
538
- const attributeId = idMapping[idn];
539
- if (!attributeId) {
540
- if (verbose)
541
- console.log(`⚠️ Skipping ${idn} - no ID mapping`);
542
- continue;
543
- }
544
- // Create attribute object for push
545
- const attributeToUpdate = {
546
- id: attributeId,
547
- idn: currentAttr.idn,
548
- value: currentAttr.value,
549
- title: currentAttr.title || "",
550
- description: currentAttr.description || "",
551
- group: currentAttr.group || "",
552
- is_hidden: currentAttr.is_hidden,
553
- possible_values: currentAttr.possible_values || [],
554
- value_type: currentAttr.value_type?.replace(/^"?AttributeValueTypes\.(.+)"?$/, '$1') || "string"
555
- };
556
- await updateCustomerAttribute(client, attributeToUpdate);
557
- attributesPushed++;
558
- if (verbose) {
559
- console.log(` ✓ Pushed changed attribute: ${idn}`);
560
- console.log(` Old value: ${backupAttr?.value || 'N/A'}`);
561
- console.log(` New value: ${currentAttr.value}`);
562
- }
563
- }
564
- }
565
- if (attributesPushed > 0) {
566
- console.log(`↑ Pushed ${attributesPushed} changed customer attributes to NEWO API`);
567
- // Show summary of what was pushed
568
- console.log(` 📊 Pushed attributes:`);
569
- for (const [idn, currentAttr] of currentAttrs) {
570
- const backupAttr = backupAttrs.get(idn);
571
- const hasChanged = !backupAttr ||
572
- currentAttr.value !== backupAttr.value ||
573
- currentAttr.title !== backupAttr.title ||
574
- currentAttr.description !== backupAttr.description ||
575
- currentAttr.group !== backupAttr.group ||
576
- currentAttr.is_hidden !== backupAttr.is_hidden;
577
- if (hasChanged) {
578
- console.log(` • ${idn}: ${currentAttr.title || 'No title'}`);
579
- console.log(` Value: ${currentAttr.value}`);
580
- }
581
- }
582
- // Update backup file after successful push
583
- await fs.writeFile(attributesBackupFile, currentContent, 'utf8');
584
- newHashes[attributesFile] = sha256(currentContent);
585
- pushed++;
586
- }
587
- else if (verbose) {
588
- console.log(` ✓ No attribute value changes detected`);
589
- }
590
- }
591
- else {
592
- console.log(`⚠️ Failed to parse attributes for comparison`);
593
- }
594
- }
595
- catch (error) {
596
- console.error(`❌ Failed to push changed attributes: ${error instanceof Error ? error.message : String(error)}`);
597
- // Don't update hash/backup on failure so it will retry next time
598
- }
599
- }
600
- else if (verbose) {
601
- console.log(` ✓ No attributes file changes`);
602
- }
603
- }
604
- else {
605
- // No backup exists, create initial backup
606
- await fs.writeFile(attributesBackupFile, currentContent, 'utf8');
607
- if (verbose)
608
- console.log(`✓ Created initial attributes backup for diff tracking`);
609
- }
610
- }
611
- else if (verbose) {
612
- console.log('ℹ️ No attributes file or ID mapping found for push checking');
613
- }
614
- }
615
- catch (error) {
616
- if (verbose)
617
- console.log(`⚠️ Attributes push check failed: ${error instanceof Error ? error.message : String(error)}`);
618
- }
619
- // Regenerate flows.yaml if metadata changed
620
- if (metadataChanged) {
621
- try {
622
- if (verbose)
623
- console.log('🔄 Metadata changed, regenerating flows.yaml...');
624
- // Create backup of current flows.yaml for format comparison
625
- const flowsFile = flowsYamlPath(customer.idn);
626
- let flowsBackup = '';
627
- if (await fs.pathExists(flowsFile)) {
628
- flowsBackup = await fs.readFile(flowsFile, 'utf8');
629
- const backupPath = `${flowsFile}.backup`;
630
- await fs.writeFile(backupPath, flowsBackup, 'utf8');
631
- if (verbose)
632
- console.log(`✓ Created flows.yaml backup at ${backupPath}`);
633
- }
634
- // Re-fetch agents for flows.yaml regeneration
635
- const agentsForFlows = [];
636
- for (const projectData of Object.values(projects)) {
637
- const projectAgents = await listAgents(client, projectData.projectId);
638
- agentsForFlows.push(...projectAgents);
639
- }
640
- // Regenerate flows.yaml
641
- await generateFlowsYaml(client, customer, agentsForFlows, verbose);
642
- // Update flows.yaml hash
643
- if (await fs.pathExists(flowsFile)) {
644
- const newFlowsContent = await fs.readFile(flowsFile, 'utf8');
645
- newHashes[flowsFile] = sha256(newFlowsContent);
646
- // Compare format with backup
647
- if (flowsBackup) {
648
- const sizeDiff = newFlowsContent.length - flowsBackup.length;
649
- if (verbose) {
650
- console.log(`✓ Regenerated flows.yaml (size change: ${sizeDiff > 0 ? '+' : ''}${sizeDiff} chars)`);
651
- }
652
- }
653
- }
654
- console.log('↑ Regenerated flows.yaml due to metadata changes');
655
- }
656
- catch (error) {
657
- console.error(`❌ Failed to regenerate flows.yaml: ${error instanceof Error ? error.message : String(error)}`);
658
- }
659
- }
660
- await saveHashes(newHashes, customer.idn);
661
- console.log(pushed ? `✅ Push complete. ${pushed} file(s) updated.` : '✅ Nothing to push.');
662
- }
663
- export async function status(customer, verbose = false) {
664
- await ensureState(customer.idn);
665
- if (!(await fs.pathExists(mapPath(customer.idn)))) {
666
- console.log(`No map for customer ${customer.idn}. Run \`newo pull --customer ${customer.idn}\` first.`);
667
- return;
668
- }
669
- if (verbose)
670
- console.log(`📋 Loading project mapping and hashes for customer ${customer.idn}...`);
671
- const idMapData = await fs.readJson(mapPath(customer.idn));
672
- const hashes = await loadHashes(customer.idn);
673
- let dirty = 0;
674
- // Handle both old single-project format and new multi-project format with type guards
675
- const projects = isProjectMap(idMapData) && idMapData.projects
676
- ? idMapData.projects
677
- : isLegacyProjectMap(idMapData)
678
- ? { '': idMapData }
679
- : (() => { throw new Error('Invalid project map format'); })();
680
- for (const [projectIdn, projectData] of Object.entries(projects)) {
681
- if (verbose && projectIdn)
682
- console.log(`📁 Checking project: ${projectIdn}`);
683
- for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
684
- if (verbose)
685
- console.log(` 📁 Checking agent: ${agentIdn}`);
686
- for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
687
- if (verbose)
688
- console.log(` 📁 Checking flow: ${flowIdn}`);
689
- for (const [skillIdn, skillMeta] of Object.entries(flowObj.skills)) {
690
- // Try new folder structure first
691
- const newPath = projectIdn ?
692
- skillScriptPath(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
693
- skillScriptPath(customer.idn, '', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
694
- // Fallback to legacy structure
695
- const legacyPath = projectIdn ?
696
- skillPath(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type) :
697
- skillPath(customer.idn, '', agentIdn, flowIdn, skillIdn, skillMeta.runner_type);
698
- let currentPath = newPath;
699
- let exists = await fs.pathExists(newPath);
700
- // If new structure doesn't exist, try legacy structure
701
- if (!exists) {
702
- exists = await fs.pathExists(legacyPath);
703
- currentPath = legacyPath;
704
- }
705
- if (!exists) {
706
- console.log(`D ${currentPath}`);
707
- dirty++;
708
- if (verbose)
709
- console.log(` ❌ Deleted: ${currentPath}`);
710
- continue;
711
- }
712
- const content = await fs.readFile(currentPath, 'utf8');
713
- const h = sha256(content);
714
- const oldHash = hashes[currentPath];
715
- if (verbose) {
716
- console.log(` 📄 ${currentPath}`);
717
- console.log(` Old hash: ${oldHash || 'none'}`);
718
- console.log(` New hash: ${h}`);
719
- }
720
- if (oldHash !== h) {
721
- console.log(`M ${currentPath}`);
722
- dirty++;
723
- if (verbose)
724
- console.log(` 🔄 Modified: ${currentPath}`);
725
- }
726
- else if (verbose) {
727
- console.log(` ✓ Unchanged: ${currentPath}`);
728
- }
729
- }
730
- // Check metadata.yaml files for changes (after skill files)
731
- for (const [skillIdn] of Object.entries(flowObj.skills)) {
732
- const metadataPath = projectIdn ?
733
- skillMetadataPath(customer.idn, projectIdn, agentIdn, flowIdn, skillIdn) :
734
- skillMetadataPath(customer.idn, '', agentIdn, flowIdn, skillIdn);
735
- if (await fs.pathExists(metadataPath)) {
736
- const metadataContent = await fs.readFile(metadataPath, 'utf8');
737
- const h = sha256(metadataContent);
738
- const oldHash = hashes[metadataPath];
739
- if (verbose) {
740
- console.log(` 📄 ${metadataPath}`);
741
- console.log(` Old hash: ${oldHash || 'none'}`);
742
- console.log(` New hash: ${h}`);
743
- }
744
- if (oldHash !== h) {
745
- console.log(`M ${metadataPath}`);
746
- dirty++;
747
- // Show which metadata fields changed
748
- try {
749
- const newMetadata = yaml.load(metadataContent);
750
- console.log(` 📊 Metadata changed for skill: ${skillIdn}`);
751
- if (newMetadata?.title) {
752
- console.log(` • Title: ${newMetadata.title}`);
753
- }
754
- if (newMetadata?.runner_type) {
755
- console.log(` • Runner: ${newMetadata.runner_type}`);
756
- }
757
- if (newMetadata?.model) {
758
- console.log(` • Model: ${newMetadata.model.provider_idn}/${newMetadata.model.model_idn}`);
759
- }
760
- }
761
- catch (e) {
762
- // Fallback to simple message
763
- if (verbose)
764
- console.log(` 🔄 Modified: metadata.yaml`);
765
- }
766
- }
767
- else if (verbose) {
768
- console.log(` ✓ Unchanged: ${metadataPath}`);
769
- }
770
- }
771
- }
772
- }
773
- }
774
- }
775
- // Check attributes file for changes
776
- try {
777
- const attributesFile = customerAttributesPath(customer.idn);
778
- if (await fs.pathExists(attributesFile)) {
779
- const content = await fs.readFile(attributesFile, 'utf8');
780
- const h = sha256(content);
781
- const oldHash = hashes[attributesFile];
782
- if (verbose) {
783
- console.log(`📄 ${attributesFile}`);
784
- console.log(` Old hash: ${oldHash || 'none'}`);
785
- console.log(` New hash: ${h}`);
786
- }
787
- if (oldHash !== h) {
788
- console.log(`M ${attributesFile}`);
789
- dirty++;
790
- // Show which attributes changed by comparing with backup
791
- try {
792
- const attributesBackupFile = customerAttributesBackupPath(customer.idn);
793
- if (await fs.pathExists(attributesBackupFile)) {
794
- const backupContent = await fs.readFile(attributesBackupFile, 'utf8');
795
- const parseYaml = (content) => {
796
- let yamlContent = content.replace(/!enum "([^"]+)"/g, '"$1"');
797
- return yaml.load(yamlContent);
798
- };
799
- const currentData = parseYaml(content);
800
- const backupData = parseYaml(backupContent);
801
- if (currentData?.attributes && backupData?.attributes) {
802
- const currentAttrs = new Map(currentData.attributes.map(attr => [attr.idn, attr]));
803
- const backupAttrs = new Map(backupData.attributes.map(attr => [attr.idn, attr]));
804
- const changedAttributes = [];
805
- for (const [idn, currentAttr] of currentAttrs) {
806
- const backupAttr = backupAttrs.get(idn);
807
- const hasChanged = !backupAttr ||
808
- currentAttr.value !== backupAttr.value ||
809
- currentAttr.title !== backupAttr.title ||
810
- currentAttr.description !== backupAttr.description ||
811
- currentAttr.group !== backupAttr.group ||
812
- currentAttr.is_hidden !== backupAttr.is_hidden;
813
- if (hasChanged) {
814
- changedAttributes.push(idn);
815
- }
816
- }
817
- if (changedAttributes.length > 0) {
818
- console.log(` 📊 Changed attributes (${changedAttributes.length}):`);
819
- changedAttributes.slice(0, 5).forEach(idn => {
820
- const current = currentAttrs.get(idn);
821
- const backup = backupAttrs.get(idn);
822
- console.log(` • ${idn}: ${current?.title || 'No title'}`);
823
- if (verbose) {
824
- console.log(` Old: ${backup?.value || 'N/A'}`);
825
- console.log(` New: ${current?.value || 'N/A'}`);
826
- }
827
- });
828
- if (changedAttributes.length > 5) {
829
- console.log(` ... and ${changedAttributes.length - 5} more`);
830
- }
831
- }
832
- }
833
- }
834
- }
835
- catch (e) {
836
- // Fallback to simple message if diff analysis fails
837
- }
838
- if (verbose)
839
- console.log(` 🔄 Modified: attributes.yaml`);
840
- }
841
- else if (verbose) {
842
- console.log(` ✓ Unchanged: attributes.yaml`);
843
- }
844
- }
845
- }
846
- catch (error) {
847
- if (verbose)
848
- console.log(`⚠️ Error checking attributes: ${error instanceof Error ? error.message : String(error)}`);
849
- }
850
- // Check flows.yaml file for changes
851
- const flowsFile = flowsYamlPath(customer.idn);
852
- if (await fs.pathExists(flowsFile)) {
853
- try {
854
- const flowsContent = await fs.readFile(flowsFile, 'utf8');
855
- const h = sha256(flowsContent);
856
- const oldHash = hashes[flowsFile];
857
- if (verbose) {
858
- console.log(`📄 flows.yaml`);
859
- console.log(` Old hash: ${oldHash || 'none'}`);
860
- console.log(` New hash: ${h}`);
861
- }
862
- if (oldHash !== h) {
863
- console.log(`M ${flowsFile}`);
864
- dirty++;
865
- if (verbose) {
866
- const flowsStats = await fs.stat(flowsFile);
867
- console.log(` 🔄 Modified: flows.yaml`);
868
- console.log(` 📊 Size: ${(flowsStats.size / 1024).toFixed(1)}KB`);
869
- console.log(` 📅 Last modified: ${flowsStats.mtime.toISOString()}`);
870
- }
871
- }
872
- else if (verbose) {
873
- const flowsStats = await fs.stat(flowsFile);
874
- console.log(` ✓ Unchanged: flows.yaml`);
875
- console.log(` 📅 Last modified: ${flowsStats.mtime.toISOString()}`);
876
- console.log(` 📊 Size: ${(flowsStats.size / 1024).toFixed(1)}KB`);
877
- }
878
- }
879
- catch (error) {
880
- if (verbose)
881
- console.log(`⚠️ Error checking flows.yaml: ${error instanceof Error ? error.message : String(error)}`);
882
- }
883
- }
884
- console.log(dirty ? `${dirty} changed file(s).` : 'Clean.');
885
- }
886
- async function generateFlowsYaml(client, customer, agents, verbose = false) {
887
- const flowsData = { flows: [] };
888
- // Calculate total flows for progress tracking
889
- const totalFlows = agents.reduce((sum, agent) => sum + (agent.flows?.length || 0), 0);
890
- let processedFlows = 0;
891
- if (!verbose && totalFlows > 0) {
892
- console.log(`📄 Generating flows.yaml (${totalFlows} flows)...`);
893
- }
894
- for (const agent of agents) {
895
- if (verbose)
896
- console.log(` 📁 Processing agent: ${agent.idn}`);
897
- const agentFlows = [];
898
- for (const flow of agent.flows ?? []) {
899
- processedFlows++;
900
- if (verbose) {
901
- console.log(` 📄 Processing flow: ${flow.idn}`);
902
- }
903
- else {
904
- // Simple progress indicator without verbose mode
905
- const percent = Math.round((processedFlows / totalFlows) * 100);
906
- const progressBar = '█'.repeat(Math.floor(percent / 5)) + '░'.repeat(20 - Math.floor(percent / 5));
907
- const progressText = ` [${progressBar}] ${percent}% (${processedFlows}/${totalFlows}) ${flow.idn}`;
908
- // Pad the line to clear any leftover text from longer previous lines
909
- const padding = ' '.repeat(Math.max(0, 80 - progressText.length));
910
- process.stdout.write(`\r${progressText}${padding}`);
911
- }
912
- // Get skills for this flow
913
- const skills = await listFlowSkills(client, flow.id);
914
- const skillsData = skills.map(skill => ({
915
- idn: skill.idn,
916
- title: skill.title || "",
917
- prompt_script: `flows/${flow.idn}/${skill.idn}.${skill.runner_type === 'nsl' ? 'jinja' : 'guidance'}`,
918
- runner_type: `!enum "RunnerType.${skill.runner_type}"`,
919
- model: {
920
- model_idn: skill.model.model_idn,
921
- provider_idn: skill.model.provider_idn
922
- },
923
- parameters: skill.parameters.map(param => ({
924
- name: param.name,
925
- default_value: param.default_value || " "
926
- }))
927
- }));
928
- // Get events for this flow
929
- let eventsData = [];
930
- try {
931
- const events = await listFlowEvents(client, flow.id);
932
- eventsData = events.map(event => ({
933
- title: event.description,
934
- idn: event.idn,
935
- skill_selector: `!enum "SkillSelector.${event.skill_selector}"`,
936
- skill_idn: event.skill_idn || null,
937
- state_idn: event.state_idn || null,
938
- integration_idn: event.integration_idn || null,
939
- connector_idn: event.connector_idn || null,
940
- interrupt_mode: `!enum "InterruptMode.${event.interrupt_mode}"`
941
- }));
942
- if (verbose)
943
- console.log(` 📋 Found ${events.length} events`);
944
- }
945
- catch (error) {
946
- if (verbose)
947
- console.log(` ⚠️ No events found for flow ${flow.idn}`);
948
- }
949
- // Get state fields for this flow
950
- let stateFieldsData = [];
951
- try {
952
- const states = await listFlowStates(client, flow.id);
953
- stateFieldsData = states.map(state => ({
954
- title: state.title,
955
- idn: state.idn,
956
- default_value: state.default_value || null,
957
- scope: `!enum "StateFieldScope.${state.scope}"`
958
- }));
959
- if (verbose)
960
- console.log(` 📊 Found ${states.length} state fields`);
961
- }
962
- catch (error) {
963
- if (verbose)
964
- console.log(` ⚠️ No state fields found for flow ${flow.idn}`);
965
- }
966
- agentFlows.push({
967
- idn: flow.idn,
968
- title: flow.title,
969
- description: flow.description || null,
970
- default_runner_type: `!enum "RunnerType.${flow.default_runner_type}"`,
971
- default_provider_idn: flow.default_model.provider_idn,
972
- default_model_idn: flow.default_model.model_idn,
973
- skills: skillsData,
974
- events: eventsData,
975
- state_fields: stateFieldsData
976
- });
977
- }
978
- const agentData = {
979
- agent_idn: agent.idn,
980
- agent_description: agent.description || null,
981
- agent_flows: agentFlows
982
- };
983
- flowsData.flows.push(agentData);
984
- }
985
- // Clear progress bar and move to new line
986
- if (!verbose && totalFlows > 0) {
987
- process.stdout.write('\n');
988
- }
989
- // Convert to YAML and write to file with custom enum handling
990
- let yamlContent = yaml.dump(flowsData, {
991
- indent: 2,
992
- lineWidth: -1,
993
- noRefs: true,
994
- sortKeys: false,
995
- quotingType: '"',
996
- forceQuotes: false,
997
- flowLevel: -1,
998
- styles: {
999
- '!!str': 'literal' // Use literal style for multiline strings
1000
- }
1001
- });
1002
- // Post-process to fix enum formatting
1003
- yamlContent = yamlContent.replace(/"(!enum \\"([^"]+)\\")"/g, '!enum "$2"');
1004
- // Post-process to fix multiline string formatting to match expected format
1005
- yamlContent = yamlContent.replace(/^(\s+agent_description: )"([^"]*)"$/gm, (match, indent, desc) => {
1006
- // Check for long descriptions that should be multiline
1007
- if (desc.length > 80 && desc.includes(' (clients of your business)')) {
1008
- // Split the ConvoAgent description into multiline YAML format
1009
- return `${indent}"${desc.replace(/(\. This Agent communicates with Users) \(clients of your business\)/, '$1\\\n \\ (clients of your business)')}"`;
1010
- }
1011
- if (desc.length > 100 && desc.includes('within a browser')) {
1012
- // Split the MagicWorker description into multiline YAML format
1013
- return `${indent}"${desc.replace(/(within a browser and behaving "like a human" when interacting with web applications that lack APIs\.) (This agent is often used)/, '$1\\\n \\ $2')}"`;
1014
- }
1015
- return match;
1016
- });
1017
- const yamlPath = flowsYamlPath(customer.idn);
1018
- await writeFileSafe(yamlPath, yamlContent);
1019
- console.log(`✓ Generated flows.yaml`);
1020
- }
1021
- // Conversation sync functions
1022
- export async function pullConversations(client, customer, options = {}, verbose = false) {
1023
- if (verbose)
1024
- console.log(`💬 Fetching conversations for customer ${customer.idn}...`);
1025
- try {
1026
- // Get all user personas with pagination
1027
- const allPersonas = [];
1028
- let page = 1;
1029
- const perPage = 50;
1030
- let hasMore = true;
1031
- while (hasMore) {
1032
- const response = await listUserPersonas(client, page, perPage);
1033
- allPersonas.push(...response.items);
1034
- if (verbose)
1035
- console.log(`📋 Page ${page}: Found ${response.items.length} personas (${allPersonas.length}/${response.metadata.total} total)`);
1036
- hasMore = response.items.length === perPage && allPersonas.length < response.metadata.total;
1037
- page++;
1038
- }
1039
- if (options.maxPersonas && allPersonas.length > options.maxPersonas) {
1040
- allPersonas.splice(options.maxPersonas);
1041
- if (verbose)
1042
- console.log(`⚠️ Limited to ${options.maxPersonas} personas as requested`);
1043
- }
1044
- if (verbose)
1045
- console.log(`👥 Processing ${allPersonas.length} personas...`);
1046
- // Process personas concurrently with limited concurrency
1047
- const processedPersonas = [];
1048
- await Promise.all(allPersonas.map(persona => concurrencyLimit(async () => {
1049
- try {
1050
- // Extract phone number from actors
1051
- const phoneActor = persona.actors.find(actor => actor.integration_idn === 'newo_voice' &&
1052
- actor.connector_idn === 'newo_voice_connector' &&
1053
- actor.contact_information?.startsWith('+'));
1054
- const phone = phoneActor?.contact_information || null;
1055
- // Get acts for this persona
1056
- const allActs = [];
1057
- let actPage = 1;
1058
- const actsPerPage = 100; // Higher limit for acts
1059
- let hasMoreActs = true;
1060
- // Get user actor IDs from persona actors first
1061
- const userActors = persona.actors.filter(actor => actor.integration_idn === 'newo_voice' &&
1062
- actor.connector_idn === 'newo_voice_connector');
1063
- if (userActors.length === 0) {
1064
- if (verbose)
1065
- console.log(` 👤 ${persona.name}: No voice actors found, skipping`);
1066
- // No voice actors, can't get chat history - add persona with empty acts
1067
- processedPersonas.push({
1068
- id: persona.id,
1069
- name: persona.name,
1070
- phone,
1071
- act_count: persona.act_count,
1072
- acts: []
1073
- });
1074
- if (verbose)
1075
- console.log(` ✓ Processed ${persona.name}: 0 acts (no voice actors)`);
1076
- return; // Return from the concurrency function
1077
- }
1078
- // Safety mechanism to prevent infinite loops
1079
- const maxPages = 50; // Limit to 50 pages (5000 acts max per persona)
1080
- while (hasMoreActs && actPage <= maxPages) {
1081
- try {
1082
- const chatHistoryParams = {
1083
- user_actor_id: userActors[0].id,
1084
- page: actPage,
1085
- per: actsPerPage
1086
- };
1087
- if (verbose)
1088
- console.log(` 📄 ${persona.name}: Fetching page ${actPage}...`);
1089
- const chatResponse = await getChatHistory(client, chatHistoryParams);
1090
- if (chatResponse.items && chatResponse.items.length > 0) {
1091
- // Convert chat history format to acts format - create minimal ConversationAct objects
1092
- const convertedActs = chatResponse.items.map((item) => ({
1093
- id: item.id || `chat_${Math.random()}`,
1094
- command_act_id: null,
1095
- external_event_id: item.external_event_id || 'chat_history',
1096
- arguments: [],
1097
- reference_idn: (item.is_agent === true) ? 'agent_message' : 'user_message',
1098
- runtime_context_id: item.runtime_context_id || 'chat_history',
1099
- source_text: item.payload?.text || item.message || item.content || item.text || '',
1100
- original_text: item.payload?.text || item.message || item.content || item.text || '',
1101
- datetime: item.datetime || item.created_at || item.timestamp || new Date().toISOString(),
1102
- user_actor_id: userActors[0].id,
1103
- agent_actor_id: null,
1104
- user_persona_id: persona.id,
1105
- user_persona_name: persona.name,
1106
- agent_persona_id: item.agent_persona_id || 'unknown',
1107
- external_id: item.external_id || null,
1108
- integration_idn: 'newo_voice',
1109
- connector_idn: 'newo_voice_connector',
1110
- to_integration_idn: null,
1111
- to_connector_idn: null,
1112
- is_agent: Boolean(item.is_agent === true),
1113
- project_idn: null,
1114
- flow_idn: item.flow_idn || 'unknown',
1115
- skill_idn: item.skill_idn || 'unknown',
1116
- session_id: item.session_id || 'unknown',
1117
- recordings: item.recordings || [],
1118
- contact_information: item.contact_information || null
1119
- }));
1120
- allActs.push(...convertedActs);
1121
- if (verbose && convertedActs.length > 0) {
1122
- console.log(` 👤 ${persona.name}: Chat History - ${convertedActs.length} messages (${allActs.length} total)`);
1123
- }
1124
- // Check if we should continue paginating
1125
- const hasMetadata = chatResponse.metadata?.total !== undefined;
1126
- const currentTotal = chatResponse.metadata?.total || 0;
1127
- hasMoreActs = chatResponse.items.length === actsPerPage &&
1128
- hasMetadata &&
1129
- allActs.length < currentTotal;
1130
- actPage++;
1131
- if (verbose)
1132
- console.log(` 📊 ${persona.name}: Page ${actPage - 1} done, ${allActs.length}/${currentTotal} total acts`);
1133
- }
1134
- else {
1135
- // No more items
1136
- hasMoreActs = false;
1137
- if (verbose)
1138
- console.log(` 📊 ${persona.name}: No more chat history items`);
1139
- }
1140
- }
1141
- catch (chatError) {
1142
- if (verbose)
1143
- console.log(` ⚠️ Chat history failed for ${persona.name}: ${chatError instanceof Error ? chatError.message : String(chatError)}`);
1144
- hasMoreActs = false;
1145
- }
1146
- }
1147
- if (actPage > maxPages) {
1148
- if (verbose)
1149
- console.log(` ⚠️ ${persona.name}: Reached max pages limit (${maxPages}), stopping pagination`);
1150
- }
1151
- // Sort acts by datetime ascending (chronological order)
1152
- allActs.sort((a, b) => new Date(a.datetime).getTime() - new Date(b.datetime).getTime());
1153
- // Process acts into simplified format - exclude redundant fields
1154
- const processedActs = allActs.map(act => {
1155
- const processedAct = {
1156
- datetime: act.datetime,
1157
- type: act.reference_idn,
1158
- message: act.source_text
1159
- };
1160
- // Only include non-redundant fields
1161
- if (act.contact_information) {
1162
- processedAct.contact_information = act.contact_information;
1163
- }
1164
- if (act.flow_idn && act.flow_idn !== 'unknown') {
1165
- processedAct.flow_idn = act.flow_idn;
1166
- }
1167
- if (act.skill_idn && act.skill_idn !== 'unknown') {
1168
- processedAct.skill_idn = act.skill_idn;
1169
- }
1170
- if (act.session_id && act.session_id !== 'unknown') {
1171
- processedAct.session_id = act.session_id;
1172
- }
1173
- return processedAct;
1174
- });
1175
- processedPersonas.push({
1176
- id: persona.id,
1177
- name: persona.name,
1178
- phone,
1179
- act_count: persona.act_count,
1180
- acts: processedActs
1181
- });
1182
- if (verbose)
1183
- console.log(` ✓ Processed ${persona.name}: ${processedActs.length} acts`);
1184
- }
1185
- catch (error) {
1186
- console.error(`❌ Failed to process persona ${persona.name}:`, error);
1187
- // Continue with other personas
1188
- }
1189
- })));
1190
- // Sort personas by most recent act time (descending) - use latest act from acts array
1191
- processedPersonas.sort((a, b) => {
1192
- const aLatestTime = a.acts.length > 0 ? a.acts[a.acts.length - 1].datetime : '1970-01-01T00:00:00.000Z';
1193
- const bLatestTime = b.acts.length > 0 ? b.acts[b.acts.length - 1].datetime : '1970-01-01T00:00:00.000Z';
1194
- return new Date(bLatestTime).getTime() - new Date(aLatestTime).getTime();
1195
- });
1196
- // Calculate totals
1197
- const totalActs = processedPersonas.reduce((sum, persona) => sum + persona.acts.length, 0);
1198
- // Create final conversations data
1199
- const conversationsData = {
1200
- personas: processedPersonas,
1201
- total_personas: processedPersonas.length,
1202
- total_acts: totalActs,
1203
- generated_at: new Date().toISOString()
1204
- };
1205
- // Save to YAML file
1206
- const conversationsPath = `newo_customers/${customer.idn}/conversations.yaml`;
1207
- const yamlContent = yaml.dump(conversationsData, {
1208
- indent: 2,
1209
- quotingType: '"',
1210
- forceQuotes: false,
1211
- lineWidth: 120,
1212
- noRefs: true,
1213
- sortKeys: false,
1214
- flowLevel: -1
1215
- });
1216
- await writeFileSafe(conversationsPath, yamlContent);
1217
- if (verbose) {
1218
- console.log(`✓ Saved conversations to ${conversationsPath}`);
1219
- console.log(`📊 Summary: ${processedPersonas.length} personas, ${totalActs} total acts`);
1220
- }
1221
- }
1222
- catch (error) {
1223
- console.error(`❌ Failed to pull conversations for ${customer.idn}:`, error);
1224
- throw error;
1225
- }
1226
- }
1
+ /**
2
+ * NEWO CLI Sync Operations - Modular architecture entry point
3
+ */
4
+ // Re-export from specialized modules
5
+ export { saveCustomerAttributes } from './sync/attributes.js';
6
+ export { pullConversations } from './sync/conversations.js';
7
+ export { status } from './sync/status.js';
8
+ export { pullSingleProject, pullAll } from './sync/projects.js';
9
+ export { pushChanged } from './sync/push.js';
10
+ export { generateFlowsYaml } from './sync/metadata.js';
11
+ // Re-export type guards for backward compatibility
12
+ export { isProjectMap, isLegacyProjectMap } from './sync/projects.js';
1227
13
  //# sourceMappingURL=sync.js.map