newo 3.6.2 → 3.7.1

package/src/sync/conversations.ts

@@ -1,8 +1,16 @@
 /**
  * Conversations synchronization module
+ *
+ * Incremental/resumable conversation pull:
+ *  - Writes per-persona JSON files to newo_customers/<idn>/conversations/<persona_id>.json as they arrive
+ *  - Updates conversations.yaml aggregate after each persona finishes
+ *  - Skips personas already fully fetched (resume support) unless --force passed via env NEWO_CONV_FORCE=1
+ *  - Graceful on partial failure: individual persona errors do not abort the batch, state is preserved
  */
 import { listUserPersonas, getChatHistory } from '../api.js';
 import { writeFileSafe } from '../fsutil.js';
+import fs from 'fs-extra';
+import path from 'path';
 import yaml from 'js-yaml';
 import pLimit from 'p-limit';
 import type { AxiosInstance } from 'axios';
@@ -16,11 +24,108 @@ import type {
   ConversationsData
 } from '../types.js';
 
-// Concurrency limit for API calls
 const concurrencyLimit = pLimit(5);
 
+type PersonaState = {
+  id: string;
+  name: string;
+  phone: string | null;
+  act_count: number;
+  acts: ProcessedAct[];
+  fetched_at: string;
+  complete: boolean;
+  last_error?: string;
+};
+
+function personaFilePath(customerIdn: string, personaId: string): string {
+  return path.join('newo_customers', customerIdn, 'conversations', `${personaId}.json`);
+}
+
+function aggregateYamlPath(customerIdn: string): string {
+  return path.join('newo_customers', customerIdn, 'conversations.yaml');
+}
+
+async function readPersonaState(customerIdn: string, personaId: string): Promise<PersonaState | null> {
+  const p = personaFilePath(customerIdn, personaId);
+  if (!(await fs.pathExists(p))) return null;
+  try {
+    return JSON.parse(await fs.readFile(p, 'utf8')) as PersonaState;
+  } catch {
+    return null;
+  }
+}
+
+async function writePersonaState(customerIdn: string, state: PersonaState): Promise<void> {
+  await writeFileSafe(personaFilePath(customerIdn, state.id), JSON.stringify(state, null, 2));
+}
+
+async function writeAggregateYaml(customerIdn: string): Promise<{ personas: number; acts: number }> {
+  const dir = path.join('newo_customers', customerIdn, 'conversations');
+  const files = (await fs.pathExists(dir)) ? await fs.readdir(dir) : [];
+  const personas: ProcessedPersona[] = [];
+  for (const f of files) {
+    if (!f.endsWith('.json')) continue;
+    try {
+      const state = JSON.parse(await fs.readFile(path.join(dir, f), 'utf8')) as PersonaState;
+      personas.push({
+        id: state.id,
+        name: state.name,
+        phone: state.phone,
+        act_count: state.act_count,
+        acts: state.acts
+      });
+    } catch {
+      // skip corrupted file
+    }
+  }
+
+  personas.sort((a, b) => {
+    const aLatestTime = a.acts.length > 0 ? a.acts[a.acts.length - 1]!.datetime : '1970-01-01T00:00:00.000Z';
+    const bLatestTime = b.acts.length > 0 ? b.acts[b.acts.length - 1]!.datetime : '1970-01-01T00:00:00.000Z';
+    return new Date(bLatestTime).getTime() - new Date(aLatestTime).getTime();
+  });
+
+  const totalActs = personas.reduce((sum, p) => sum + p.acts.length, 0);
+
+  const data: ConversationsData = {
+    personas,
+    total_personas: personas.length,
+    total_acts: totalActs,
+    generated_at: new Date().toISOString()
+  };
+
+  const yamlContent = yaml.dump(data, {
+    indent: 2,
+    quotingType: '"',
+    forceQuotes: false,
+    lineWidth: 120,
+    noRefs: true,
+    sortKeys: false,
+    flowLevel: -1
+  });
+
+  await writeFileSafe(aggregateYamlPath(customerIdn), yamlContent);
+  return { personas: personas.length, acts: totalActs };
+}
+
+function buildProcessedActs(raw: ConversationAct[]): ProcessedAct[] {
+  const sorted = [...raw].sort((a, b) => new Date(a.datetime).getTime() - new Date(b.datetime).getTime());
+  return sorted.map(act => {
+    const processedAct: ProcessedAct = {
+      datetime: act.datetime,
+      type: act.reference_idn,
+      message: act.source_text
+    };
+    if (act.contact_information) (processedAct as any).contact_information = act.contact_information;
+    if (act.flow_idn && act.flow_idn !== 'unknown') (processedAct as any).flow_idn = act.flow_idn;
+    if (act.skill_idn && act.skill_idn !== 'unknown') (processedAct as any).skill_idn = act.skill_idn;
+    if (act.session_id && act.session_id !== 'unknown') (processedAct as any).session_id = act.session_id;
+    return processedAct;
+  });
+}
+
 /**
- * Pull conversations for a customer and save to YAML
+ * Pull conversations for a customer and save incrementally.
  */
 export async function pullConversations(
   client: AxiosInstance,
@@ -28,230 +133,178 @@ export async function pullConversations(
   options: ConversationOptions = {},
   verbose: boolean = false
 ): Promise<void> {
-  if (verbose) console.log(`💬 Fetching conversations for customer ${customer.idn}...`);
+  const force = process.env.NEWO_CONV_FORCE === '1';
+  console.log(`💬 Fetching conversations for ${customer.idn}${force ? ' (force re-fetch)' : ' (resume mode)'}...`);
+
+  // Ensure output dirs exist
+  await fs.ensureDir(path.join('newo_customers', customer.idn, 'conversations'));
+
+  // 1. Enumerate all personas
+  const allPersonas: UserPersona[] = [];
+  let page = 1;
+  const perPage = 50;
+  let hasMore = true;
+
+  while (hasMore) {
+    const response = await listUserPersonas(client, page, perPage);
+    allPersonas.push(...response.items);
+    if (verbose) console.log(`📋 Page ${page}: ${response.items.length} personas (${allPersonas.length}/${response.metadata.total})`);
+    hasMore = response.items.length === perPage && allPersonas.length < response.metadata.total;
+    page++;
+  }
 
-  try {
-    // Get all user personas with pagination
-    const allPersonas: UserPersona[] = [];
-    let page = 1;
-    const perPage = 50;
-    let hasMore = true;
+  if (options.maxPersonas && allPersonas.length > options.maxPersonas) {
+    allPersonas.splice(options.maxPersonas);
+  }
 
-    while (hasMore) {
-      const response = await listUserPersonas(client, page, perPage);
-      allPersonas.push(...response.items);
+  const total = allPersonas.length;
+  console.log(`👥 Found ${total} personas. Processing with concurrency=5...`);
+
+  let done = 0;
+  let skipped = 0;
+  let failed = 0;
+
+  await Promise.all(allPersonas.map(persona => concurrencyLimit(async () => {
+    try {
+      // Resume: skip if already complete
+      const existing = await readPersonaState(customer.idn, persona.id);
+      if (!force && existing && existing.complete) {
+        skipped++;
+        done++;
+        if (verbose) console.log(`⏭️  [${done}/${total}] ${persona.name}: already complete (${existing.acts.length} acts)`);
+        return;
+      }
 
-      if (verbose) console.log(`📋 Page ${page}: Found ${response.items.length} personas (${allPersonas.length}/${response.metadata.total} total)`);
+      const phoneActor = persona.actors.find(actor =>
+        actor.integration_idn === 'newo_voice' &&
+        actor.connector_idn === 'newo_voice_connector' &&
+        actor.contact_information?.startsWith('+')
+      );
+      const phone = phoneActor?.contact_information || null;
 
-      hasMore = response.items.length === perPage && allPersonas.length < response.metadata.total;
-      page++;
-    }
+      const userActors = persona.actors.filter(actor =>
+        actor.integration_idn === 'newo_voice' &&
+        actor.connector_idn === 'newo_voice_connector'
+      );
 
-    if (options.maxPersonas && allPersonas.length > options.maxPersonas) {
-      allPersonas.splice(options.maxPersonas);
-      if (verbose) console.log(`⚠️  Limited to ${options.maxPersonas} personas as requested`);
-    }
+      if (userActors.length === 0) {
+        const state: PersonaState = {
+          id: persona.id,
+          name: persona.name,
+          phone,
+          act_count: persona.act_count,
+          acts: [],
+          fetched_at: new Date().toISOString(),
+          complete: true
+        };
+        await writePersonaState(customer.idn, state);
+        done++;
+        if (verbose) console.log(`✓ [${done}/${total}] ${persona.name}: no voice actors`);
+        return;
+      }
 
-    if (verbose) console.log(`👥 Processing ${allPersonas.length} personas...`);
-
-    // Process personas concurrently with limited concurrency
-    const processedPersonas: ProcessedPersona[] = [];
-
-    await Promise.all(allPersonas.map(persona => concurrencyLimit(async () => {
-      try {
-        // Extract phone number from actors
-        const phoneActor = persona.actors.find(actor =>
-          actor.integration_idn === 'newo_voice' &&
-          actor.connector_idn === 'newo_voice_connector' &&
-          actor.contact_information?.startsWith('+')
-        );
-        const phone = phoneActor?.contact_information || null;
-
-        // Get acts for this persona
-        const allActs: ConversationAct[] = [];
-        let actPage = 1;
-        const actsPerPage = 100; // Higher limit for acts
-        let hasMoreActs = true;
-
-        // Get user actor IDs from persona actors first
-        const userActors = persona.actors.filter(actor =>
-          actor.integration_idn === 'newo_voice' &&
-          actor.connector_idn === 'newo_voice_connector'
-        );
-
-        if (userActors.length === 0) {
-          if (verbose) console.log(`  👤 ${persona.name}: No voice actors found, skipping`);
-          // No voice actors, can't get chat history - add persona with empty acts
-          processedPersonas.push({
-            id: persona.id,
-            name: persona.name,
-            phone,
-            act_count: persona.act_count,
-            acts: []
+      // Fetch acts paginated
+      const allActs: ConversationAct[] = [];
+      let actPage = 1;
+      const actsPerPage = 100;
+      let hasMoreActs = true;
+      const maxPages = 50;
+      let lastError: string | undefined;
+
+      while (hasMoreActs && actPage <= maxPages) {
+        try {
+          const chatResponse = await getChatHistory(client, {
+            user_actor_id: userActors[0]!.id,
+            page: actPage,
+            per: actsPerPage
           });
-          if (verbose) console.log(`  ✓ Processed ${persona.name}: 0 acts (no voice actors)`);
-          return; // Return from the concurrency function
-        }
-
-        // Safety mechanism to prevent infinite loops
-        const maxPages = 50; // Limit to 50 pages (5000 acts max per persona)
 
-        while (hasMoreActs && actPage <= maxPages) {
-          try {
-            const chatHistoryParams = {
+          if (chatResponse.items && chatResponse.items.length > 0) {
+            const convertedActs: ConversationAct[] = chatResponse.items.map((item: any) => ({
+              id: item.id || `chat_${Math.random()}`,
+              command_act_id: null,
+              external_event_id: item.external_event_id || 'chat_history',
+              arguments: [],
+              reference_idn: (item.is_agent === true) ? 'agent_message' : 'user_message',
+              runtime_context_id: item.runtime_context_id || 'chat_history',
+              source_text: item.payload?.text || item.message || item.content || item.text || '',
+              original_text: item.payload?.text || item.message || item.content || item.text || '',
+              datetime: item.datetime || item.created_at || item.timestamp || new Date().toISOString(),
               user_actor_id: userActors[0]!.id,
-              page: actPage,
-              per: actsPerPage
+              agent_actor_id: null,
+              user_persona_id: persona.id,
+              user_persona_name: persona.name,
+              agent_persona_id: item.agent_persona_id || 'unknown',
+              external_id: item.external_id || null,
+              integration_idn: 'newo_voice',
+              connector_idn: 'newo_voice_connector',
+              to_integration_idn: null,
+              to_connector_idn: null,
+              is_agent: Boolean(item.is_agent === true),
+              project_idn: null,
+              flow_idn: item.flow_idn || 'unknown',
+              skill_idn: item.skill_idn || 'unknown',
+              session_id: item.session_id || 'unknown',
+              recordings: item.recordings || [],
+              contact_information: item.contact_information || null
+            }));
+
+            allActs.push(...convertedActs);
+
+            // Save partial progress every page
+            const partialState: PersonaState = {
+              id: persona.id,
+              name: persona.name,
+              phone,
+              act_count: persona.act_count,
+              acts: buildProcessedActs(allActs),
+              fetched_at: new Date().toISOString(),
+              complete: false
             };
+            await writePersonaState(customer.idn, partialState);
 
-            if (verbose) console.log(`    📄 ${persona.name}: Fetching page ${actPage}...`);
-            const chatResponse = await getChatHistory(client, chatHistoryParams);
-
-            if (chatResponse.items && chatResponse.items.length > 0) {
-                // Convert chat history format to acts format - create minimal ConversationAct objects
-                const convertedActs: ConversationAct[] = chatResponse.items.map((item: any) => ({
-                  id: item.id || `chat_${Math.random()}`,
-                  command_act_id: null,
-                  external_event_id: item.external_event_id || 'chat_history',
-                  arguments: [],
-                  reference_idn: (item.is_agent === true) ? 'agent_message' : 'user_message',
-                  runtime_context_id: item.runtime_context_id || 'chat_history',
-                  source_text: item.payload?.text || item.message || item.content || item.text || '',
-                  original_text: item.payload?.text || item.message || item.content || item.text || '',
-                  datetime: item.datetime || item.created_at || item.timestamp || new Date().toISOString(),
-                  user_actor_id: userActors[0]!.id,
-                  agent_actor_id: null,
-                  user_persona_id: persona.id,
-                  user_persona_name: persona.name,
-                  agent_persona_id: item.agent_persona_id || 'unknown',
-                  external_id: item.external_id || null,
-                  integration_idn: 'newo_voice',
-                  connector_idn: 'newo_voice_connector',
-                  to_integration_idn: null,
-                  to_connector_idn: null,
-                  is_agent: Boolean(item.is_agent === true),
-                  project_idn: null,
-                  flow_idn: item.flow_idn || 'unknown',
-                  skill_idn: item.skill_idn || 'unknown',
-                  session_id: item.session_id || 'unknown',
-                  recordings: item.recordings || [],
-                  contact_information: item.contact_information || null
-                }));
-
-                allActs.push(...convertedActs);
-
-                if (verbose && convertedActs.length > 0) {
-                  console.log(`  👤 ${persona.name}: Chat History - ${convertedActs.length} messages (${allActs.length} total)`);
-                }
-
-                // Check if we should continue paginating
-                const hasMetadata = chatResponse.metadata?.total !== undefined;
-                const currentTotal = chatResponse.metadata?.total || 0;
-
-                hasMoreActs = chatResponse.items.length === actsPerPage &&
-                             hasMetadata &&
-                             allActs.length < currentTotal;
-
-                actPage++;
-
-                if (verbose) console.log(`    📊 ${persona.name}: Page ${actPage - 1} done, ${allActs.length}/${currentTotal} total acts`);
-              } else {
-                // No more items
-                hasMoreActs = false;
-                if (verbose) console.log(`    📊 ${persona.name}: No more chat history items`);
-              }
-          } catch (chatError) {
-            if (verbose) console.log(`  ⚠️  Chat history failed for ${persona.name}: ${chatError instanceof Error ? chatError.message : String(chatError)}`);
+            const currentTotal = chatResponse.metadata?.total || 0;
+            hasMoreActs = chatResponse.items.length === actsPerPage && allActs.length < currentTotal;
+            actPage++;
+          } else {
             hasMoreActs = false;
           }
+        } catch (chatError) {
+          lastError = chatError instanceof Error ? chatError.message : String(chatError);
+          if (verbose) console.log(`⚠️  ${persona.name} page ${actPage}: ${lastError}`);
+          hasMoreActs = false;
         }
-
-        if (actPage > maxPages) {
-          if (verbose) console.log(`  ⚠️  ${persona.name}: Reached max pages limit (${maxPages}), stopping pagination`);
-        }
-
-        // Sort acts by datetime ascending (chronological order)
-        allActs.sort((a, b) => new Date(a.datetime).getTime() - new Date(b.datetime).getTime());
-
-        // Process acts into simplified format - exclude redundant fields
-        const processedActs: ProcessedAct[] = allActs.map(act => {
-          const processedAct: ProcessedAct = {
-            datetime: act.datetime,
-            type: act.reference_idn,
-            message: act.source_text
-          };
-
-          // Only include non-redundant fields
-          if (act.contact_information) {
-            (processedAct as any).contact_information = act.contact_information;
-          }
-          if (act.flow_idn && act.flow_idn !== 'unknown') {
-            (processedAct as any).flow_idn = act.flow_idn;
-          }
-          if (act.skill_idn && act.skill_idn !== 'unknown') {
-            (processedAct as any).skill_idn = act.skill_idn;
-          }
-          if (act.session_id && act.session_id !== 'unknown') {
-            (processedAct as any).session_id = act.session_id;
-          }
-
-          return processedAct;
-        });
-
-        processedPersonas.push({
-          id: persona.id,
-          name: persona.name,
-          phone,
-          act_count: persona.act_count,
-          acts: processedActs
-        });
-
-        if (verbose) console.log(`  ✓ Processed ${persona.name}: ${processedActs.length} acts`);
-      } catch (error) {
-        console.error(`❌ Failed to process persona ${persona.name}:`, error);
-        // Continue with other personas
       }
-    })));
-
-    // Sort personas by most recent act time (descending) - use latest act from acts array
-    processedPersonas.sort((a, b) => {
-      const aLatestTime = a.acts.length > 0 ? a.acts[a.acts.length - 1]!.datetime : '1970-01-01T00:00:00.000Z';
-      const bLatestTime = b.acts.length > 0 ? b.acts[b.acts.length - 1]!.datetime : '1970-01-01T00:00:00.000Z';
-      return new Date(bLatestTime).getTime() - new Date(aLatestTime).getTime();
-    });
-
-    // Calculate totals
-    const totalActs = processedPersonas.reduce((sum, persona) => sum + persona.acts.length, 0);
-
-    // Create final conversations data
-    const conversationsData: ConversationsData = {
-      personas: processedPersonas,
-      total_personas: processedPersonas.length,
-      total_acts: totalActs,
-      generated_at: new Date().toISOString()
-    };
 
-    // Save to YAML file
-    const conversationsPath = `newo_customers/${customer.idn}/conversations.yaml`;
-    const yamlContent = yaml.dump(conversationsData, {
-      indent: 2,
-      quotingType: '"',
-      forceQuotes: false,
-      lineWidth: 120,
-      noRefs: true,
-      sortKeys: false,
-      flowLevel: -1
-    });
-
-    await writeFileSafe(conversationsPath, yamlContent);
-
-    if (verbose) {
-      console.log(`✓ Saved conversations to ${conversationsPath}`);
-      console.log(`📊 Summary: ${processedPersonas.length} personas, ${totalActs} total acts`);
+      const finalState: PersonaState = {
+        id: persona.id,
+        name: persona.name,
+        phone,
+        act_count: persona.act_count,
+        acts: buildProcessedActs(allActs),
+        fetched_at: new Date().toISOString(),
+        complete: !lastError
+      };
+      if (lastError) finalState.last_error = lastError;
+      await writePersonaState(customer.idn, finalState);
+
+      // Incremental YAML aggregate every persona
+      const agg = await writeAggregateYaml(customer.idn);
+
+      done++;
+      if (lastError) failed++;
+      console.log(`${lastError ? '⚠️ ' : '✓'} [${done}/${total}] ${persona.name}: ${finalState.acts.length} acts${lastError ? ` (partial: ${lastError})` : ''} | total so far: ${agg.personas} personas / ${agg.acts} acts`);
+    } catch (error) {
+      failed++;
+      done++;
+      const msg = error instanceof Error ? error.message : String(error);
+      console.error(`❌ [${done}/${total}] ${persona.name}: ${msg}`);
     }
-
-  } catch (error) {
-    console.error(`❌ Failed to pull conversations for ${customer.idn}:`, error);
-    throw error;
-  }
-}
+  })));
+
+  // Final aggregate write
+  const final = await writeAggregateYaml(customer.idn);
+  console.log(`\n✅ Done. ${final.personas} personas, ${final.acts} acts. Skipped ${skipped} (already cached), ${failed} had errors.`);
+  console.log(`   Aggregate: ${aggregateYamlPath(customer.idn)}`);
+  console.log(`   Per-persona: newo_customers/${customer.idn}/conversations/<id>.json`);
+}

package/src/sync/json-attr-utils.ts

@@ -0,0 +1,95 @@
+/**
+ * JSON-typed attribute helpers.
+ *
+ * Why this exists:
+ *
+ * The NEWO platform stores some attributes (e.g.
+ * `project_attributes_private_dynamic_workflow_builder_canvas`) as
+ * `value_type: json`. The API may return the `value` field as either a
+ * STRING containing JSON or as an already-parsed OBJECT.
+ *
+ * Without normalization, two bugs leak through:
+ *
+ * 1. When the API returns the value as an OBJECT, `yaml.dump` serializes
+ *    it as a YAML structure (mappings/sequences). Pushing back then sends
+ *    `{"value": {...}}` instead of `{"value": "..."}`, breaking the
+ *    Workflow Builder which expects the canvas as a JSON STRING.
+ *
+ * 2. The push-time change check used `String(localAttr.value)` for
+ *    comparison. With objects this collapses to `"[object Object]"` on
+ *    both sides — silently masking real changes — and with mismatched
+ *    string vs object representations it triggers spurious pushes that
+ *    overwrite the canvas with the wrong shape (Builder shows blank).
+ *
+ * The fix is conservative: for `value_type: json` only, always coerce the
+ * value to a STRING when persisting and when pushing, and use canonical
+ * JSON for comparisons. String-typed values in the wild are left
+ * untouched, so no churn for the majority of attributes.
+ */
+
+/**
+ * True if the attribute is a JSON-typed attribute (case- and
+ * format-insensitive: handles `json`, `JSON`, `AttributeValueTypes.json`,
+ * `ValueType.JSON`, etc.).
+ */
+export function isJsonValueType(valueType: unknown): boolean {
+  if (typeof valueType !== 'string') return false;
+  const lower = valueType.toLowerCase();
+  return lower === 'json' || lower.endsWith('.json');
+}
+
+/**
+ * Coerce a JSON-typed attribute's value to a STRING suitable for storage
+ * in attributes.yaml and for sending to the platform.
+ *
+ * - `null` / `undefined` → `''`
+ * - object → compact JSON string (`JSON.stringify(value)`)
+ * - string → returned as-is (we trust the platform's existing format)
+ * - other → `String(value)`
+ *
+ * We deliberately do NOT re-format string values, even when they look
+ * like JSON. Many existing canvases are stored pretty-printed and
+ * reformatting would create huge spurious diffs in users' repos.
+ */
+export function normalizeJsonValueForStorage(value: unknown): string {
+  if (value == null) return '';
+  if (typeof value === 'string') return value;
+  if (typeof value === 'object') {
+    try {
+      return JSON.stringify(value);
+    } catch {
+      return String(value);
+    }
+  }
+  return String(value);
+}
+
+/**
+ * Canonical comparison for JSON-typed attribute values.
+ *
+ * Returns the canonical form (compact JSON if parseable, otherwise the
+ * raw string). Use this on both sides of a comparison so that pretty- vs
+ * compact-printed JSON does not register as a change, and so that an
+ * object on one side equals its stringified form on the other side.
+ */
+export function canonicalJsonValue(value: unknown): string {
+  const stringified = normalizeJsonValueForStorage(value);
+  if (stringified === '') return '';
+  try {
+    return JSON.stringify(JSON.parse(stringified));
+  } catch {
+    return stringified;
+  }
+}
+
+/**
+ * True if two JSON-typed attribute values are semantically equal.
+ *
+ * Handles the four mismatched representations that can occur during a
+ * pull/push cycle:
+ *   string vs string (different whitespace/indent), object vs string,
+ *   string vs object, object vs object.
+ */
+export function jsonValuesEqual(a: unknown, b: unknown): boolean {
+  return canonicalJsonValue(a) === canonicalJsonValue(b);
+}