newo 3.6.1 → 3.7.0

package/src/lint/reporters/text.ts

@@ -0,0 +1,58 @@
+/**
+ * Human-readable terminal reporter. Mirrors the ESLint 'stylish' layout:
+ *
+ *   path/to/file.jinja
+ *     12:5   error   Unknown skill: foo. Did you mean: bar?  E100
+ *     ...
+ *
+ *   2 problems (1 error, 1 warning)
+ */
+import path from 'path';
+import type { ProjectLintReport, LintResult } from 'newo-dsl-analyzer';
+import type { Reporter } from './types.js';
+
+const RED = '\x1b[31m';
+const YELLOW = '\x1b[33m';
+const CYAN = '\x1b[36m';
+const GREY = '\x1b[90m';
+const RESET = '\x1b[0m';
+const BOLD = '\x1b[1m';
+
+export const textReporter: Reporter = {
+  write(report: ProjectLintReport): string {
+    const lines: string[] = [];
+    const filesWithIssues = report.results.filter(r => r.diagnostics.length > 0);
+
+    for (const result of filesWithIssues) {
+      lines.push(renderFile(result));
+      lines.push('');
+    }
+
+    lines.push(renderSummary(report));
+    return lines.join('\n');
+  },
+};
+
+function renderFile(result: LintResult): string {
+  const rel = path.relative(process.cwd(), result.filePath);
+  const header = `${BOLD}${CYAN}${rel}${RESET}`;
+  const rows = result.diagnostics.map(d => {
+    const loc = `${d.range.start.line}:${d.range.start.column}`;
+    const sev = d.severity === 'error'
+      ? `${RED}error${RESET}`
+      : d.severity === 'warning'
+        ? `${YELLOW}warning${RESET}`
+        : `${GREY}${d.severity}${RESET}`;
+    return `  ${loc.padEnd(7)} ${sev.padEnd(16)} ${d.message}  ${GREY}${d.code}${RESET}`;
+  });
+  return [header, ...rows].join('\n');
+}
+
+function renderSummary(report: ProjectLintReport): string {
+  const total = report.errorCount + report.warningCount;
+  if (total === 0) {
+    return `${GREY}No issues found.${RESET}`;
+  }
+  const color = report.errorCount > 0 ? RED : YELLOW;
+  return `${color}${BOLD}${total} problems${RESET} (${report.errorCount} error${report.errorCount === 1 ? '' : 's'}, ${report.warningCount} warning${report.warningCount === 1 ? '' : 's'})`;
+}

package/src/lint/reporters/types.ts

@@ -0,0 +1,7 @@
+import type { ProjectLintReport } from 'newo-dsl-analyzer';
+
+export type ReporterName = 'text' | 'json' | 'sarif';
+
+export interface Reporter {
+  write(report: ProjectLintReport): string;
+}

package/src/sync/attributes.ts

@@ -11,6 +11,7 @@ import {
 import path from 'path';
 import fs from 'fs-extra';
 import yaml from 'js-yaml';
+import { patchYamlToPyyaml } from '../format/yaml-patch.js';
 import type { AxiosInstance } from 'axios';
 import type { CustomerConfig } from '../types.js';
 
@@ -72,25 +73,24 @@ export async function saveCustomerAttributes(
       attributes: cleanAttributes
     };
 
-    // Configure YAML output to match reference format exactly
+    // Emit YAML without folding/wrapping; patchYamlToPyyaml handles long-line
+    // wrapping and converts JSON-like double-quoted values to single-quoted
+    // (so strings containing `"` stay valid YAML on reload).
     let yamlContent = yaml.dump(attributesYaml, {
       indent: 2,
       quotingType: '"',
       forceQuotes: false,
-      lineWidth: 80, // Wrap long lines to match reference format
+      lineWidth: -1,
       noRefs: true,
       sortKeys: false,
-      flowLevel: -1, // Never use flow syntax
-      styles: {
-        '!!str': 'folded' // Use folded style for better line wrapping of long strings
-      }
+      flowLevel: -1,
     });
 
-    // Post-process to fix enum format and improve JSON string formatting
+    // Post-process to fix enum format
     yamlContent = yamlContent.replace(/__ENUM_PLACEHOLDER_(\w+)__/g, '!enum "AttributeValueTypes.$1"');
 
-    // Fix JSON string formatting to match reference (remove escape characters)
-    yamlContent = yamlContent.replace(/\\"/g, '"');
+    // Convert JSON-like double-quoted values to single-quoted and wrap long lines
+    yamlContent = patchYamlToPyyaml(yamlContent);
 
     // Save all files: attributes.yaml, ID mapping, and backup for diff tracking
     await writeFileSafe(customerAttributesPath(customer.idn), yamlContent);
@@ -172,20 +172,20 @@ export async function saveProjectAttributes(
       attributes: cleanAttributes
     };
 
-    // Configure YAML output
+    // Emit YAML without folding/wrapping; patchYamlToPyyaml handles long-line
+    // wrapping and converts JSON-like double-quoted values to single-quoted.
     let yamlContent = yaml.dump(attributesYaml, {
       indent: 2,
       quotingType: '"',
       forceQuotes: false,
-      lineWidth: 80,
+      lineWidth: -1,
       noRefs: true,
       sortKeys: false,
-      flowLevel: -1
+      flowLevel: -1,
     });
 
-    // Post-process to fix enum format
     yamlContent = yamlContent.replace(/__ENUM_PLACEHOLDER_(\w+)__/g, '!enum "AttributeValueTypes.$1"');
-    yamlContent = yamlContent.replace(/\\"/g, '"');
+    yamlContent = patchYamlToPyyaml(yamlContent);
 
     // Save to project directory
     const customerDir = path.join(process.cwd(), 'newo_customers', customer.idn);

package/src/sync/conversations.ts

@@ -1,8 +1,16 @@
 /**
  * Conversations synchronization module
+ *
+ * Incremental/resumable conversation pull:
+ *  - Writes per-persona JSON files to newo_customers/<idn>/conversations/<persona_id>.json as they arrive
+ *  - Updates conversations.yaml aggregate after each persona finishes
+ *  - Skips personas already fully fetched (resume support) unless --force passed via env NEWO_CONV_FORCE=1
+ *  - Graceful on partial failure: individual persona errors do not abort the batch, state is preserved
  */
 import { listUserPersonas, getChatHistory } from '../api.js';
 import { writeFileSafe } from '../fsutil.js';
+import fs from 'fs-extra';
+import path from 'path';
 import yaml from 'js-yaml';
 import pLimit from 'p-limit';
 import type { AxiosInstance } from 'axios';
@@ -16,11 +24,108 @@ import type {
   ConversationsData
 } from '../types.js';
 
-// Concurrency limit for API calls
 const concurrencyLimit = pLimit(5);
 
+type PersonaState = {
+  id: string;
+  name: string;
+  phone: string | null;
+  act_count: number;
+  acts: ProcessedAct[];
+  fetched_at: string;
+  complete: boolean;
+  last_error?: string;
+};
+
+function personaFilePath(customerIdn: string, personaId: string): string {
+  return path.join('newo_customers', customerIdn, 'conversations', `${personaId}.json`);
+}
+
+function aggregateYamlPath(customerIdn: string): string {
+  return path.join('newo_customers', customerIdn, 'conversations.yaml');
+}
+
+async function readPersonaState(customerIdn: string, personaId: string): Promise<PersonaState | null> {
+  const p = personaFilePath(customerIdn, personaId);
+  if (!(await fs.pathExists(p))) return null;
+  try {
+    return JSON.parse(await fs.readFile(p, 'utf8')) as PersonaState;
+  } catch {
+    return null;
+  }
+}
+
+async function writePersonaState(customerIdn: string, state: PersonaState): Promise<void> {
+  await writeFileSafe(personaFilePath(customerIdn, state.id), JSON.stringify(state, null, 2));
+}
+
+async function writeAggregateYaml(customerIdn: string): Promise<{ personas: number; acts: number }> {
+  const dir = path.join('newo_customers', customerIdn, 'conversations');
+  const files = (await fs.pathExists(dir)) ? await fs.readdir(dir) : [];
+  const personas: ProcessedPersona[] = [];
+  for (const f of files) {
+    if (!f.endsWith('.json')) continue;
+    try {
+      const state = JSON.parse(await fs.readFile(path.join(dir, f), 'utf8')) as PersonaState;
+      personas.push({
+        id: state.id,
+        name: state.name,
+        phone: state.phone,
+        act_count: state.act_count,
+        acts: state.acts
+      });
+    } catch {
+      // skip corrupted file
+    }
+  }
+
+  personas.sort((a, b) => {
+    const aLatestTime = a.acts.length > 0 ? a.acts[a.acts.length - 1]!.datetime : '1970-01-01T00:00:00.000Z';
+    const bLatestTime = b.acts.length > 0 ? b.acts[b.acts.length - 1]!.datetime : '1970-01-01T00:00:00.000Z';
+    return new Date(bLatestTime).getTime() - new Date(aLatestTime).getTime();
+  });
+
+  const totalActs = personas.reduce((sum, p) => sum + p.acts.length, 0);
+
+  const data: ConversationsData = {
+    personas,
+    total_personas: personas.length,
+    total_acts: totalActs,
+    generated_at: new Date().toISOString()
+  };
+
+  const yamlContent = yaml.dump(data, {
+    indent: 2,
+    quotingType: '"',
+    forceQuotes: false,
+    lineWidth: 120,
+    noRefs: true,
+    sortKeys: false,
+    flowLevel: -1
+  });
+
+  await writeFileSafe(aggregateYamlPath(customerIdn), yamlContent);
+  return { personas: personas.length, acts: totalActs };
+}
+
+function buildProcessedActs(raw: ConversationAct[]): ProcessedAct[] {
+  const sorted = [...raw].sort((a, b) => new Date(a.datetime).getTime() - new Date(b.datetime).getTime());
+  return sorted.map(act => {
+    const processedAct: ProcessedAct = {
+      datetime: act.datetime,
+      type: act.reference_idn,
+      message: act.source_text
+    };
+    if (act.contact_information) (processedAct as any).contact_information = act.contact_information;
+    if (act.flow_idn && act.flow_idn !== 'unknown') (processedAct as any).flow_idn = act.flow_idn;
+    if (act.skill_idn && act.skill_idn !== 'unknown') (processedAct as any).skill_idn = act.skill_idn;
+    if (act.session_id && act.session_id !== 'unknown') (processedAct as any).session_id = act.session_id;
+    return processedAct;
+  });
+}
+
 /**
- * Pull conversations for a customer and save to YAML
+ * Pull conversations for a customer and save incrementally.
  */
 export async function pullConversations(
   client: AxiosInstance,
@@ -28,230 +133,178 @@ export async function pullConversations(
   options: ConversationOptions = {},
   verbose: boolean = false
 ): Promise<void> {
-  if (verbose) console.log(`💬 Fetching conversations for customer ${customer.idn}...`);
+  const force = process.env.NEWO_CONV_FORCE === '1';
+  console.log(`💬 Fetching conversations for ${customer.idn}${force ? ' (force re-fetch)' : ' (resume mode)'}...`);
+
+  // Ensure output dirs exist
+  await fs.ensureDir(path.join('newo_customers', customer.idn, 'conversations'));
+
+  // 1. Enumerate all personas
+  const allPersonas: UserPersona[] = [];
+  let page = 1;
+  const perPage = 50;
+  let hasMore = true;
+
+  while (hasMore) {
+    const response = await listUserPersonas(client, page, perPage);
+    allPersonas.push(...response.items);
+    if (verbose) console.log(`📋 Page ${page}: ${response.items.length} personas (${allPersonas.length}/${response.metadata.total})`);
+    hasMore = response.items.length === perPage && allPersonas.length < response.metadata.total;
+    page++;
+  }
 
-  try {
-    // Get all user personas with pagination
-    const allPersonas: UserPersona[] = [];
-    let page = 1;
-    const perPage = 50;
-    let hasMore = true;
+  if (options.maxPersonas && allPersonas.length > options.maxPersonas) {
+    allPersonas.splice(options.maxPersonas);
+  }
 
-    while (hasMore) {
-      const response = await listUserPersonas(client, page, perPage);
-      allPersonas.push(...response.items);
+  const total = allPersonas.length;
+  console.log(`👥 Found ${total} personas. Processing with concurrency=5...`);
+
+  let done = 0;
+  let skipped = 0;
+  let failed = 0;
+
+  await Promise.all(allPersonas.map(persona => concurrencyLimit(async () => {
+    try {
+      // Resume: skip if already complete
+      const existing = await readPersonaState(customer.idn, persona.id);
+      if (!force && existing && existing.complete) {
+        skipped++;
+        done++;
+        if (verbose) console.log(`⏭️  [${done}/${total}] ${persona.name}: already complete (${existing.acts.length} acts)`);
+        return;
+      }
 
-      if (verbose) console.log(`📋 Page ${page}: Found ${response.items.length} personas (${allPersonas.length}/${response.metadata.total} total)`);
+      const phoneActor = persona.actors.find(actor =>
+        actor.integration_idn === 'newo_voice' &&
+        actor.connector_idn === 'newo_voice_connector' &&
+        actor.contact_information?.startsWith('+')
+      );
+      const phone = phoneActor?.contact_information || null;
 
-      hasMore = response.items.length === perPage && allPersonas.length < response.metadata.total;
-      page++;
-    }
+      const userActors = persona.actors.filter(actor =>
+        actor.integration_idn === 'newo_voice' &&
+        actor.connector_idn === 'newo_voice_connector'
+      );
 
-    if (options.maxPersonas && allPersonas.length > options.maxPersonas) {
-      allPersonas.splice(options.maxPersonas);
-      if (verbose) console.log(`⚠️  Limited to ${options.maxPersonas} personas as requested`);
-    }
+      if (userActors.length === 0) {
+        const state: PersonaState = {
+          id: persona.id,
+          name: persona.name,
+          phone,
+          act_count: persona.act_count,
+          acts: [],
+          fetched_at: new Date().toISOString(),
+          complete: true
+        };
+        await writePersonaState(customer.idn, state);
+        done++;
+        if (verbose) console.log(`✓ [${done}/${total}] ${persona.name}: no voice actors`);
+        return;
+      }
 
-    if (verbose) console.log(`👥 Processing ${allPersonas.length} personas...`);
-
-    // Process personas concurrently with limited concurrency
-    const processedPersonas: ProcessedPersona[] = [];
-
-    await Promise.all(allPersonas.map(persona => concurrencyLimit(async () => {
-      try {
-        // Extract phone number from actors
-        const phoneActor = persona.actors.find(actor =>
-          actor.integration_idn === 'newo_voice' &&
-          actor.connector_idn === 'newo_voice_connector' &&
-          actor.contact_information?.startsWith('+')
-        );
-        const phone = phoneActor?.contact_information || null;
-
-        // Get acts for this persona
-        const allActs: ConversationAct[] = [];
-        let actPage = 1;
-        const actsPerPage = 100; // Higher limit for acts
-        let hasMoreActs = true;
-
-        // Get user actor IDs from persona actors first
-        const userActors = persona.actors.filter(actor =>
-          actor.integration_idn === 'newo_voice' &&
-          actor.connector_idn === 'newo_voice_connector'
-        );
-
-        if (userActors.length === 0) {
-          if (verbose) console.log(`  👤 ${persona.name}: No voice actors found, skipping`);
-          // No voice actors, can't get chat history - add persona with empty acts
-          processedPersonas.push({
-            id: persona.id,
-            name: persona.name,
-            phone,
-            act_count: persona.act_count,
-            acts: []
+      // Fetch acts paginated
+      const allActs: ConversationAct[] = [];
+      let actPage = 1;
+      const actsPerPage = 100;
+      let hasMoreActs = true;
+      const maxPages = 50;
+      let lastError: string | undefined;
+
+      while (hasMoreActs && actPage <= maxPages) {
+        try {
+          const chatResponse = await getChatHistory(client, {
+            user_actor_id: userActors[0]!.id,
+            page: actPage,
+            per: actsPerPage
           });
-          if (verbose) console.log(`  ✓ Processed ${persona.name}: 0 acts (no voice actors)`);
-          return; // Return from the concurrency function
-        }
-
-        // Safety mechanism to prevent infinite loops
-        const maxPages = 50; // Limit to 50 pages (5000 acts max per persona)
 
-        while (hasMoreActs && actPage <= maxPages) {
-          try {
-            const chatHistoryParams = {
+          if (chatResponse.items && chatResponse.items.length > 0) {
+            const convertedActs: ConversationAct[] = chatResponse.items.map((item: any) => ({
+              id: item.id || `chat_${Math.random()}`,
+              command_act_id: null,
+              external_event_id: item.external_event_id || 'chat_history',
+              arguments: [],
+              reference_idn: (item.is_agent === true) ? 'agent_message' : 'user_message',
+              runtime_context_id: item.runtime_context_id || 'chat_history',
+              source_text: item.payload?.text || item.message || item.content || item.text || '',
+              original_text: item.payload?.text || item.message || item.content || item.text || '',
+              datetime: item.datetime || item.created_at || item.timestamp || new Date().toISOString(),
               user_actor_id: userActors[0]!.id,
-              page: actPage,
-              per: actsPerPage
+              agent_actor_id: null,
+              user_persona_id: persona.id,
+              user_persona_name: persona.name,
+              agent_persona_id: item.agent_persona_id || 'unknown',
+              external_id: item.external_id || null,
+              integration_idn: 'newo_voice',
+              connector_idn: 'newo_voice_connector',
+              to_integration_idn: null,
+              to_connector_idn: null,
+              is_agent: Boolean(item.is_agent === true),
+              project_idn: null,
+              flow_idn: item.flow_idn || 'unknown',
+              skill_idn: item.skill_idn || 'unknown',
+              session_id: item.session_id || 'unknown',
+              recordings: item.recordings || [],
+              contact_information: item.contact_information || null
+            }));
+
+            allActs.push(...convertedActs);
+
+            // Save partial progress every page
+            const partialState: PersonaState = {
+              id: persona.id,
+              name: persona.name,
+              phone,
+              act_count: persona.act_count,
+              acts: buildProcessedActs(allActs),
+              fetched_at: new Date().toISOString(),
+              complete: false
             };
+            await writePersonaState(customer.idn, partialState);
 
-            if (verbose) console.log(`    📄 ${persona.name}: Fetching page ${actPage}...`);
-            const chatResponse = await getChatHistory(client, chatHistoryParams);
-
-            if (chatResponse.items && chatResponse.items.length > 0) {
-                // Convert chat history format to acts format - create minimal ConversationAct objects
-                const convertedActs: ConversationAct[] = chatResponse.items.map((item: any) => ({
-                  id: item.id || `chat_${Math.random()}`,
-                  command_act_id: null,
-                  external_event_id: item.external_event_id || 'chat_history',
-                  arguments: [],
-                  reference_idn: (item.is_agent === true) ? 'agent_message' : 'user_message',
-                  runtime_context_id: item.runtime_context_id || 'chat_history',
-                  source_text: item.payload?.text || item.message || item.content || item.text || '',
-                  original_text: item.payload?.text || item.message || item.content || item.text || '',
-                  datetime: item.datetime || item.created_at || item.timestamp || new Date().toISOString(),
-                  user_actor_id: userActors[0]!.id,
-                  agent_actor_id: null,
-                  user_persona_id: persona.id,
-                  user_persona_name: persona.name,
-                  agent_persona_id: item.agent_persona_id || 'unknown',
-                  external_id: item.external_id || null,
-                  integration_idn: 'newo_voice',
-                  connector_idn: 'newo_voice_connector',
-                  to_integration_idn: null,
-                  to_connector_idn: null,
-                  is_agent: Boolean(item.is_agent === true),
-                  project_idn: null,
-                  flow_idn: item.flow_idn || 'unknown',
-                  skill_idn: item.skill_idn || 'unknown',
-                  session_id: item.session_id || 'unknown',
-                  recordings: item.recordings || [],
-                  contact_information: item.contact_information || null
-                }));
-
-                allActs.push(...convertedActs);
-
-                if (verbose && convertedActs.length > 0) {
-                  console.log(`  👤 ${persona.name}: Chat History - ${convertedActs.length} messages (${allActs.length} total)`);
-                }
-
-                // Check if we should continue paginating
-                const hasMetadata = chatResponse.metadata?.total !== undefined;
-                const currentTotal = chatResponse.metadata?.total || 0;
-
-                hasMoreActs = chatResponse.items.length === actsPerPage &&
-                             hasMetadata &&
-                             allActs.length < currentTotal;
-
-                actPage++;
-
-                if (verbose) console.log(`    📊 ${persona.name}: Page ${actPage - 1} done, ${allActs.length}/${currentTotal} total acts`);
-              } else {
-                // No more items
-                hasMoreActs = false;
-                if (verbose) console.log(`    📊 ${persona.name}: No more chat history items`);
-              }
-          } catch (chatError) {
-            if (verbose) console.log(`  ⚠️  Chat history failed for ${persona.name}: ${chatError instanceof Error ? chatError.message : String(chatError)}`);
+            const currentTotal = chatResponse.metadata?.total || 0;
+            hasMoreActs = chatResponse.items.length === actsPerPage && allActs.length < currentTotal;
+            actPage++;
+          } else {
             hasMoreActs = false;
           }
+        } catch (chatError) {
+          lastError = chatError instanceof Error ? chatError.message : String(chatError);
+          if (verbose) console.log(`⚠️  ${persona.name} page ${actPage}: ${lastError}`);
+          hasMoreActs = false;
         }
-
-        if (actPage > maxPages) {
-          if (verbose) console.log(`  ⚠️  ${persona.name}: Reached max pages limit (${maxPages}), stopping pagination`);
-        }
-
-        // Sort acts by datetime ascending (chronological order)
-        allActs.sort((a, b) => new Date(a.datetime).getTime() - new Date(b.datetime).getTime());
-
-        // Process acts into simplified format - exclude redundant fields
-        const processedActs: ProcessedAct[] = allActs.map(act => {
-          const processedAct: ProcessedAct = {
-            datetime: act.datetime,
-            type: act.reference_idn,
-            message: act.source_text
-          };
-
-          // Only include non-redundant fields
-          if (act.contact_information) {
-            (processedAct as any).contact_information = act.contact_information;
-          }
-          if (act.flow_idn && act.flow_idn !== 'unknown') {
-            (processedAct as any).flow_idn = act.flow_idn;
-          }
-          if (act.skill_idn && act.skill_idn !== 'unknown') {
-            (processedAct as any).skill_idn = act.skill_idn;
-          }
-          if (act.session_id && act.session_id !== 'unknown') {
-            (processedAct as any).session_id = act.session_id;
-          }
-
-          return processedAct;
-        });
-
-        processedPersonas.push({
-          id: persona.id,
-          name: persona.name,
-          phone,
-          act_count: persona.act_count,
-          acts: processedActs
-        });
-
-        if (verbose) console.log(`  ✓ Processed ${persona.name}: ${processedActs.length} acts`);
-      } catch (error) {
-        console.error(`❌ Failed to process persona ${persona.name}:`, error);
-        // Continue with other personas
       }
-    })));
-
-    // Sort personas by most recent act time (descending) - use latest act from acts array
-    processedPersonas.sort((a, b) => {
-      const aLatestTime = a.acts.length > 0 ? a.acts[a.acts.length - 1]!.datetime : '1970-01-01T00:00:00.000Z';
-      const bLatestTime = b.acts.length > 0 ? b.acts[b.acts.length - 1]!.datetime : '1970-01-01T00:00:00.000Z';
-      return new Date(bLatestTime).getTime() - new Date(aLatestTime).getTime();
-    });
-
-    // Calculate totals
-    const totalActs = processedPersonas.reduce((sum, persona) => sum + persona.acts.length, 0);
-
-    // Create final conversations data
-    const conversationsData: ConversationsData = {
-      personas: processedPersonas,
-      total_personas: processedPersonas.length,
-      total_acts: totalActs,
-      generated_at: new Date().toISOString()
-    };
 
-    // Save to YAML file
-    const conversationsPath = `newo_customers/${customer.idn}/conversations.yaml`;
-    const yamlContent = yaml.dump(conversationsData, {
-      indent: 2,
-      quotingType: '"',
-      forceQuotes: false,
-      lineWidth: 120,
-      noRefs: true,
-      sortKeys: false,
-      flowLevel: -1
-    });
-
-    await writeFileSafe(conversationsPath, yamlContent);
-
-    if (verbose) {
-      console.log(`✓ Saved conversations to ${conversationsPath}`);
-      console.log(`📊 Summary: ${processedPersonas.length} personas, ${totalActs} total acts`);
+      const finalState: PersonaState = {
+        id: persona.id,
+        name: persona.name,
+        phone,
+        act_count: persona.act_count,
+        acts: buildProcessedActs(allActs),
+        fetched_at: new Date().toISOString(),
+        complete: !lastError
+      };
+      if (lastError) finalState.last_error = lastError;
+      await writePersonaState(customer.idn, finalState);
+
+      // Incremental YAML aggregate every persona
+      const agg = await writeAggregateYaml(customer.idn);
+
+      done++;
+      if (lastError) failed++;
+      console.log(`${lastError ? '⚠️ ' : '✓'} [${done}/${total}] ${persona.name}: ${finalState.acts.length} acts${lastError ? ` (partial: ${lastError})` : ''} | total so far: ${agg.personas} personas / ${agg.acts} acts`);
+    } catch (error) {
+      failed++;
+      done++;
+      const msg = error instanceof Error ? error.message : String(error);
+      console.error(`❌ [${done}/${total}] ${persona.name}: ${msg}`);
     }
-
-  } catch (error) {
-    console.error(`❌ Failed to pull conversations for ${customer.idn}:`, error);
-    throw error;
-  }
-}
+  })));
+
+  // Final aggregate write
+  const final = await writeAggregateYaml(customer.idn);
+  console.log(`\n✅ Done. ${final.personas} personas, ${final.acts} acts. Skipped ${skipped} (already cached), ${failed} had errors.`);
+  console.log(`   Aggregate: ${aggregateYamlPath(customer.idn)}`);
+  console.log(`   Per-persona: newo_customers/${customer.idn}/conversations/<id>.json`);
+}