newo 3.7.0 → 3.7.2

package/src/sync/flow-metadata.ts

@@ -0,0 +1,345 @@
+/**
+ * Flow metadata sync — reconciles local flow metadata.yaml with the platform.
+ *
+ * Closes the gap behind GH issue #3 (push wiping flow event subscriptions and
+ * title): before this module existed, push only updated skill scripts. Local
+ * edits to flow title, events, or state_fields silently never reached the
+ * platform; new events created via `newo create-event` had no path to flow
+ * therefore disappeared from local metadata.yaml after a subsequent pull.
+ *
+ * Reconciliation strategy per flow (only runs when metadata.yaml hash changed):
+ *  - Compare local FlowMetadata against fresh GET responses from the platform
+ *  - Update flow title/description/runner via PATCH /api/v1/designer/flows/{id}
+ *  - For each child collection (events, state_fields):
+ *      • idn present locally + missing remotely → create
+ *      • idn present in both, contents differ   → update
+ *      • idn missing locally + present remotely → delete
+ *
+ * Hash-gating is critical: if the user never touched metadata.yaml, we never
+ * compute a remote diff, which means a stale or partially-pulled tree cannot
+ * accidentally wipe events that were created out-of-band via the Builder UI.
+ */
+import type { AxiosInstance } from 'axios';
+import {
+  listFlowEvents,
+  listFlowStates,
+  createFlowEvent,
+  updateFlowEvent,
+  deleteFlowEvent,
+  createFlowState,
+  updateFlowState,
+  deleteFlowState,
+  updateFlow
+} from '../api.js';
+import type {
+  FlowMetadata,
+  FlowEvent,
+  FlowState,
+  CreateFlowEventRequest,
+  UpdateFlowEventRequest,
+  CreateFlowStateRequest,
+  UpdateFlowStateRequest,
+  UpdateFlowRequest
+} from '../types.js';
+
+export interface FlowMetadataSyncCounts {
+  flowsUpdated: number;
+  eventsCreated: number;
+  eventsUpdated: number;
+  eventsDeleted: number;
+  statesCreated: number;
+  statesUpdated: number;
+  statesDeleted: number;
+  errors: string[];
+}
+
+export function emptyFlowSyncCounts(): FlowMetadataSyncCounts {
+  return {
+    flowsUpdated: 0,
+    eventsCreated: 0,
+    eventsUpdated: 0,
+    eventsDeleted: 0,
+    statesCreated: 0,
+    statesUpdated: 0,
+    statesDeleted: 0,
+    errors: []
+  };
+}
+
+/**
+ * True when remote FlowEvent fields differ from what the local metadata says.
+ * We only compare semantic fields the platform stores - `id` is platform-owned.
+ */
+export function flowEventDiffers(local: FlowEvent, remote: FlowEvent): boolean {
+  return (
+    normalizeStr(local.description) !== normalizeStr(remote.description) ||
+    normalizeStr(local.skill_selector) !== normalizeStr(remote.skill_selector) ||
+    normalizeStr(local.skill_idn) !== normalizeStr(remote.skill_idn) ||
+    normalizeStr(local.state_idn) !== normalizeStr(remote.state_idn) ||
+    normalizeStr(local.interrupt_mode) !== normalizeStr(remote.interrupt_mode) ||
+    normalizeStr(local.integration_idn) !== normalizeStr(remote.integration_idn) ||
+    normalizeStr(local.connector_idn) !== normalizeStr(remote.connector_idn)
+  );
+}
+
+export function flowStateDiffers(local: FlowState, remote: FlowState): boolean {
+  return (
+    normalizeStr(local.title) !== normalizeStr(remote.title) ||
+    normalizeStr(local.default_value) !== normalizeStr(remote.default_value) ||
+    normalizeStr(local.scope) !== normalizeStr(remote.scope)
+  );
+}
+
+/**
+ * Normalizes nullable/undefined string fields so YAML round-trips don't
+ * register as differences. `null`, `undefined`, and missing all collapse to ''.
+ */
+function normalizeStr(value: unknown): string {
+  if (value === null || value === undefined) return '';
+  return String(value);
+}
+
+function buildEventCreateRequest(local: FlowEvent): CreateFlowEventRequest {
+  return {
+    idn: local.idn,
+    description: local.description ?? '',
+    skill_selector: local.skill_selector,
+    ...(local.skill_idn != null ? { skill_idn: local.skill_idn } : {}),
+    state_idn: local.state_idn ?? null,
+    interrupt_mode: local.interrupt_mode,
+    integration_idn: local.integration_idn ?? '',
+    connector_idn: local.connector_idn ?? ''
+  };
+}
+
+function buildEventUpdateRequest(local: FlowEvent): UpdateFlowEventRequest {
+  return {
+    idn: local.idn,
+    description: local.description ?? '',
+    skill_selector: local.skill_selector,
+    skill_idn: local.skill_idn ?? null,
+    state_idn: local.state_idn ?? null,
+    interrupt_mode: local.interrupt_mode,
+    integration_idn: local.integration_idn ?? null,
+    connector_idn: local.connector_idn ?? null
+  };
+}
+
+function buildStateCreateRequest(local: FlowState): CreateFlowStateRequest {
+  const req: CreateFlowStateRequest = {
+    title: local.title || local.idn,
+    idn: local.idn,
+    scope: local.scope
+  };
+  if (local.default_value != null) {
+    req.default_value = local.default_value;
+  }
+  return req;
+}
+
+function buildStateUpdateRequest(local: FlowState): UpdateFlowStateRequest {
+  const req: UpdateFlowStateRequest = {
+    title: local.title || local.idn,
+    idn: local.idn,
+    scope: local.scope
+  };
+  if (local.default_value != null) {
+    req.default_value = local.default_value;
+  }
+  return req;
+}
+
+function shouldUpdateFlow(local: FlowMetadata, remote: { title: string; description: string | null; default_runner_type?: string }): boolean {
+  return (
+    normalizeStr(local.title) !== normalizeStr(remote.title) ||
+    normalizeStr(local.description) !== normalizeStr(remote.description) ||
+    normalizeStr(local.default_runner_type) !== normalizeStr(remote.default_runner_type)
+  );
+}
+
+/**
+ * Reconcile one flow's metadata with the platform.
+ *
+ * @param client          authenticated Axios client
+ * @param flowId          platform flow ID (UUID)
+ * @param local           parsed FlowMetadata from the customer's local YAML
+ * @param remoteFlow      flow data fetched from GET /flows/{id} - if null,
+ *                        flow-level updates are skipped (still syncs children).
+ *                        Pass null when caller already knows the GET endpoint
+ *                        will 404 (e.g. legacy data) or wants children-only.
+ * @param verbose         when true, emits per-operation log lines
+ * @param counts          shared counter mutated in place across multiple flows
+ */
+export async function syncFlowMetadata(
+  client: AxiosInstance,
+  flowId: string,
+  local: FlowMetadata,
+  remoteFlow: { title: string; description: string | null; default_runner_type?: string } | null,
+  verbose: boolean,
+  counts: FlowMetadataSyncCounts
+): Promise<void> {
+  // 1. Flow-level fields (title, description, runner type)
+  if (remoteFlow && shouldUpdateFlow(local, remoteFlow)) {
+    try {
+      const updateRequest: UpdateFlowRequest = {
+        idn: local.idn,
+        title: local.title,
+        description: local.description ?? '',
+        default_runner_type: local.default_runner_type,
+        default_model: local.default_model
+      };
+      await updateFlow(client, flowId, updateRequest);
+      counts.flowsUpdated++;
+      if (verbose) {
+        console.log(`    ↑ Updated flow metadata: ${local.idn} (title: "${remoteFlow.title}" → "${local.title}")`);
+      }
+    } catch (error: unknown) {
+      const msg = error instanceof Error ? error.message : String(error);
+      counts.errors.push(`Failed to update flow ${local.idn}: ${msg}`);
+      console.error(`    ❌ Failed to update flow ${local.idn}: ${msg}`);
+    }
+  }
+
+  // 2. Events
+  let remoteEvents: FlowEvent[] = [];
+  try {
+    remoteEvents = await listFlowEvents(client, flowId);
+  } catch (error: unknown) {
+    const msg = error instanceof Error ? error.message : String(error);
+    counts.errors.push(`Failed to list events for flow ${local.idn}: ${msg}`);
+    return;
+  }
+
+  const localEvents = local.events ?? [];
+  const remoteByIdn = new Map(remoteEvents.map(e => [e.idn, e]));
+  const localByIdn = new Map(localEvents.map(e => [e.idn, e]));
+
+  // Create or update events present locally
+  for (const localEvent of localEvents) {
+    const remote = remoteByIdn.get(localEvent.idn);
+    if (!remote) {
+      // Create
+      try {
+        await createFlowEvent(client, flowId, buildEventCreateRequest(localEvent));
+        counts.eventsCreated++;
+        if (verbose) console.log(`    ↑ Created event: ${local.idn}/${localEvent.idn}`);
+      } catch (error: unknown) {
+        const msg = error instanceof Error ? error.message : String(error);
+        counts.errors.push(`Failed to create event ${localEvent.idn} in flow ${local.idn}: ${msg}`);
+        console.error(`    ❌ Failed to create event ${localEvent.idn}: ${msg}`);
+      }
+    } else if (flowEventDiffers(localEvent, remote)) {
+      try {
+        await updateFlowEvent(client, remote.id, buildEventUpdateRequest(localEvent));
+        counts.eventsUpdated++;
+        if (verbose) console.log(`    ↑ Updated event: ${local.idn}/${localEvent.idn}`);
+      } catch (error: unknown) {
+        const msg = error instanceof Error ? error.message : String(error);
+        counts.errors.push(`Failed to update event ${localEvent.idn} in flow ${local.idn}: ${msg}`);
+        console.error(`    ❌ Failed to update event ${localEvent.idn}: ${msg}`);
+      }
+    }
+  }
+
+  // Delete events present remotely but missing locally
+  for (const remoteEvent of remoteEvents) {
+    if (!localByIdn.has(remoteEvent.idn)) {
+      try {
+        await deleteFlowEvent(client, remoteEvent.id);
+        counts.eventsDeleted++;
+        if (verbose) console.log(`    ↑ Deleted event: ${local.idn}/${remoteEvent.idn}`);
+      } catch (error: unknown) {
+        const msg = error instanceof Error ? error.message : String(error);
+        counts.errors.push(`Failed to delete event ${remoteEvent.idn} in flow ${local.idn}: ${msg}`);
+        console.error(`    ❌ Failed to delete event ${remoteEvent.idn}: ${msg}`);
+      }
+    }
+  }
+
+  // 3. State fields
+  let remoteStates: FlowState[] = [];
+  try {
+    remoteStates = await listFlowStates(client, flowId);
+  } catch (error: unknown) {
+    const msg = error instanceof Error ? error.message : String(error);
+    counts.errors.push(`Failed to list states for flow ${local.idn}: ${msg}`);
+    return;
+  }
+
+  const localStates = local.state_fields ?? [];
+  const remoteStatesByIdn = new Map(remoteStates.map(s => [s.idn, s]));
+  const localStatesByIdn = new Map(localStates.map(s => [s.idn, s]));
+
+  for (const localState of localStates) {
+    const remote = remoteStatesByIdn.get(localState.idn);
+    if (!remote) {
+      try {
+        await createFlowState(client, flowId, buildStateCreateRequest(localState));
+        counts.statesCreated++;
+        if (verbose) console.log(`    ↑ Created state: ${local.idn}/${localState.idn}`);
+      } catch (error: unknown) {
+        const msg = error instanceof Error ? error.message : String(error);
+        counts.errors.push(`Failed to create state ${localState.idn} in flow ${local.idn}: ${msg}`);
+        console.error(`    ❌ Failed to create state ${localState.idn}: ${msg}`);
+      }
+    } else if (flowStateDiffers(localState, remote)) {
+      try {
+        await updateFlowState(client, remote.id, buildStateUpdateRequest(localState));
+        counts.statesUpdated++;
+        if (verbose) console.log(`    ↑ Updated state: ${local.idn}/${localState.idn}`);
+      } catch (error: unknown) {
+        const msg = error instanceof Error ? error.message : String(error);
+        counts.errors.push(`Failed to update state ${localState.idn} in flow ${local.idn}: ${msg}`);
+        console.error(`    ❌ Failed to update state ${localState.idn}: ${msg}`);
+      }
+    }
+  }
+
+  for (const remoteState of remoteStates) {
+    if (!localStatesByIdn.has(remoteState.idn)) {
+      try {
+        await deleteFlowState(client, remoteState.id);
+        counts.statesDeleted++;
+        if (verbose) console.log(`    ↑ Deleted state: ${local.idn}/${remoteState.idn}`);
+      } catch (error: unknown) {
+        const msg = error instanceof Error ? error.message : String(error);
+        counts.errors.push(`Failed to delete state ${remoteState.idn} in flow ${local.idn}: ${msg}`);
+        console.error(`    ❌ Failed to delete state ${remoteState.idn}: ${msg}`);
+      }
+    }
+  }
+}
+
+/**
+ * Combined count of operations across all categories.
+ */
+export function totalFlowSyncOps(counts: FlowMetadataSyncCounts): number {
+  return (
+    counts.flowsUpdated +
+    counts.eventsCreated + counts.eventsUpdated + counts.eventsDeleted +
+    counts.statesCreated + counts.statesUpdated + counts.statesDeleted
+  );
+}
+
+/**
+ * Human-readable summary line for the push report.
+ */
+export function describeFlowSyncCounts(counts: FlowMetadataSyncCounts): string {
+  const parts: string[] = [];
+  if (counts.flowsUpdated) parts.push(`${counts.flowsUpdated} flow(s)`);
+  if (counts.eventsCreated || counts.eventsUpdated || counts.eventsDeleted) {
+    const eventOps: string[] = [];
+    if (counts.eventsCreated) eventOps.push(`+${counts.eventsCreated}`);
+    if (counts.eventsUpdated) eventOps.push(`~${counts.eventsUpdated}`);
+    if (counts.eventsDeleted) eventOps.push(`-${counts.eventsDeleted}`);
+    parts.push(`events ${eventOps.join('/')}`);
+  }
+  if (counts.statesCreated || counts.statesUpdated || counts.statesDeleted) {
+    const stateOps: string[] = [];
+    if (counts.statesCreated) stateOps.push(`+${counts.statesCreated}`);
+    if (counts.statesUpdated) stateOps.push(`~${counts.statesUpdated}`);
+    if (counts.statesDeleted) stateOps.push(`-${counts.statesDeleted}`);
+    parts.push(`states ${stateOps.join('/')}`);
+  }
+  return parts.join(', ');
+}

package/src/sync/json-attr-utils.ts

@@ -0,0 +1,95 @@
+/**
+ * JSON-typed attribute helpers.
+ *
+ * Why this exists:
+ *
+ * The NEWO platform stores some attributes (e.g.
+ * `project_attributes_private_dynamic_workflow_builder_canvas`) as
+ * `value_type: json`. The API may return the `value` field as either a
+ * STRING containing JSON or as an already-parsed OBJECT.
+ *
+ * Without normalization, two bugs leak through:
+ *
+ * 1. When the API returns the value as an OBJECT, `yaml.dump` serializes
+ *    it as a YAML structure (mappings/sequences). Pushing back then sends
+ *    `{"value": {...}}` instead of `{"value": "..."}`, breaking the
+ *    Workflow Builder which expects the canvas as a JSON STRING.
+ *
+ * 2. The push-time change check used `String(localAttr.value)` for
+ *    comparison. With objects this collapses to `"[object Object]"` on
+ *    both sides — silently masking real changes — and with mismatched
+ *    string vs object representations it triggers spurious pushes that
+ *    overwrite the canvas with the wrong shape (Builder shows blank).
+ *
+ * The fix is conservative: for `value_type: json` only, always coerce the
+ * value to a STRING when persisting and when pushing, and use canonical
+ * JSON for comparisons. String-typed values in the wild are left
+ * untouched, so no churn for the majority of attributes.
+ */
+
+/**
+ * True if the attribute is a JSON-typed attribute (case- and
+ * format-insensitive: handles `json`, `JSON`, `AttributeValueTypes.json`,
+ * `ValueType.JSON`, etc.).
+ */
+export function isJsonValueType(valueType: unknown): boolean {
+  if (typeof valueType !== 'string') return false;
+  const lower = valueType.toLowerCase();
+  return lower === 'json' || lower.endsWith('.json');
+}
+
+/**
+ * Coerce a JSON-typed attribute's value to a STRING suitable for storage
+ * in attributes.yaml and for sending to the platform.
+ *
+ * - `null` / `undefined` → `''`
+ * - object → compact JSON string (`JSON.stringify(value)`)
+ * - string → returned as-is (we trust the platform's existing format)
+ * - other → `String(value)`
+ *
+ * We deliberately do NOT re-format string values, even when they look
+ * like JSON. Many existing canvases are stored pretty-printed and
+ * reformatting would create huge spurious diffs in users' repos.
+ */
+export function normalizeJsonValueForStorage(value: unknown): string {
+  if (value == null) return '';
+  if (typeof value === 'string') return value;
+  if (typeof value === 'object') {
+    try {
+      return JSON.stringify(value);
+    } catch {
+      return String(value);
+    }
+  }
+  return String(value);
+}
+
+/**
+ * Canonical comparison for JSON-typed attribute values.
+ *
+ * Returns the canonical form (compact JSON if parseable, otherwise the
+ * raw string). Use this on both sides of a comparison so that pretty- vs
+ * compact-printed JSON does not register as a change, and so that an
+ * object on one side equals its stringified form on the other side.
+ */
+export function canonicalJsonValue(value: unknown): string {
+  const stringified = normalizeJsonValueForStorage(value);
+  if (stringified === '') return '';
+  try {
+    return JSON.stringify(JSON.parse(stringified));
+  } catch {
+    return stringified;
+  }
+}
+
+/**
+ * True if two JSON-typed attribute values are semantically equal.
+ *
+ * Handles the four mismatched representations that can occur during a
+ * pull/push cycle:
+ *   string vs string (different whitespace/indent), object vs string,
+ *   string vs object, object vs object.
+ */
+export function jsonValuesEqual(a: unknown, b: unknown): boolean {
+  return canonicalJsonValue(a) === canonicalJsonValue(b);
+}

package/src/sync/push.ts

@@ -1,13 +1,14 @@
 /**
  * Push operations for changed files
  */
-import { updateSkill, createAgent, createFlow, createSkill, publishFlow } from '../api.js';
+import { updateSkill, createAgent, createFlow, createSkill, publishFlow, getFlow } from '../api.js';
 import {
   ensureState,
   mapPath,
   skillMetadataPath,
   projectDir,
-  agentMetadataPath
+  agentMetadataPath,
+  flowMetadataPath
 } from '../fsutil.js';
 import {
   validateSkillFolder,
@@ -21,6 +22,12 @@ import { generateFlowsYaml } from './metadata.js';
 import { isProjectMap, isLegacyProjectMap } from './projects.js';
 import { flowsYamlPath } from '../fsutil.js';
 import { pushAllProjectAttributes } from './attributes.js';
+import {
+  syncFlowMetadata,
+  emptyFlowSyncCounts,
+  totalFlowSyncOps,
+  describeFlowSyncCounts
+} from './flow-metadata.js';
 import type { AxiosInstance } from 'axios';
 import type {
   ProjectData,
@@ -506,6 +513,72 @@ export async function pushChanged(client: AxiosInstance, customer: CustomerConfi
     }
   }
 
+  // Sync flow metadata (title, events, state_fields) for any flow whose
+  // metadata.yaml hash has changed. This closes the loop on GH issue #3:
+  // previously push only updated skill scripts, leaving local edits to
+  // flow events and title silently un-synced.
+  const flowSyncCounts = emptyFlowSyncCounts();
+  for (const [projectIdn, projectData] of Object.entries(projects)) {
+    for (const [agentIdn, agentObj] of Object.entries(projectData.agents)) {
+      for (const [flowIdn, flowObj] of Object.entries(agentObj.flows)) {
+        if (!flowObj.id) continue;
+
+        const metaPath = flowMetadataPath(customer.idn, projectIdn, agentIdn, flowIdn);
+        if (!(await fs.pathExists(metaPath))) continue;
+
+        const metaContent = await fs.readFile(metaPath, 'utf8');
+        const metaHash = sha256(metaContent);
+        const oldHash = hashes[metaPath];
+
+        if (oldHash === metaHash) {
+          if (verbose) console.log(`    ✓ Flow metadata unchanged: ${flowIdn}`);
+          continue;
+        }
+
+        if (verbose) console.log(`    🔄 Flow metadata changed, syncing: ${agentIdn}/${flowIdn}`);
+
+        let localFlow: FlowMetadata;
+        try {
+          localFlow = yaml.load(metaContent) as FlowMetadata;
+        } catch (error) {
+          console.error(`❌ Failed to parse flow metadata for ${flowIdn}: ${error instanceof Error ? error.message : String(error)}`);
+          continue;
+        }
+
+        let remoteFlow = null;
+        try {
+          remoteFlow = await getFlow(client, flowObj.id);
+        } catch (error: any) {
+          // 404 means the flow ID is stale; skip flow-level update but still
+          // try to sync children since list endpoints may still work.
+          if (verbose) {
+            console.log(`    ⚠️  Could not GET flow ${flowIdn}: ${error.response?.status ?? error.message}`);
+          }
+        }
+
+        const opsBefore = totalFlowSyncOps(flowSyncCounts);
+        await syncFlowMetadata(client, flowObj.id, localFlow, remoteFlow, verbose, flowSyncCounts);
+        const opsAfter = totalFlowSyncOps(flowSyncCounts);
+
+        if (opsAfter > opsBefore) {
+          pushed += (opsAfter - opsBefore);
+          metadataChanged = true;
+        }
+        // Hash is updated regardless of whether ops happened, so we don't
+        // re-scan the same untouched flow on the next push.
+        newHashes[metaPath] = metaHash;
+      }
+    }
+  }
+
+  const totalFlowOps = totalFlowSyncOps(flowSyncCounts);
+  if (totalFlowOps > 0) {
+    console.log(`↑ Flow metadata synced: ${describeFlowSyncCounts(flowSyncCounts)}`);
+  }
+  if (flowSyncCounts.errors.length > 0) {
+    console.warn(`⚠️  ${flowSyncCounts.errors.length} flow-metadata error(s) during push.`);
+  }
+
   if (verbose) console.log(`🔄 Scanned ${scanned} files, found ${pushed} changes`);
 
   // Push project attributes for all projects

package/src/types.ts

@@ -521,6 +521,23 @@ export interface CreateFlowEventResponse {
   id: string;
 }
 
+/**
+ * Payload for PATCH /api/v1/designer/flows/events/{eventId}
+ *
+ * Required fields per probe testing: idn, skill_selector, interrupt_mode.
+ * Sending the full event body is the platform's expected shape.
+ */
+export interface UpdateFlowEventRequest {
+  idn: string;
+  description?: string | null;
+  skill_selector: string;
+  skill_idn?: string | null;
+  state_idn?: string | null;
+  interrupt_mode: string;
+  integration_idn?: string | null;
+  connector_idn?: string | null;
+}
+
 export interface CreateFlowStateRequest {
   title: string;
   idn: string;
@@ -532,6 +549,29 @@ export interface CreateFlowStateResponse {
   id: string;
 }
 
+/**
+ * Payload for PUT /api/v1/designer/flows/states/{stateId}
+ */
+export interface UpdateFlowStateRequest {
+  title: string;
+  idn: string;
+  default_value?: string;
+  scope: string;
+}
+
+/**
+ * Payload for PATCH /api/v1/designer/flows/{flowId}
+ *
+ * Empty body returns 500; the platform requires the full descriptor.
+ */
+export interface UpdateFlowRequest {
+  idn: string;
+  title: string;
+  description?: string;
+  default_runner_type: RunnerType;
+  default_model: ModelConfig;
+}
+
 export interface CreateSkillParameterRequest {
   name: string;
   default_value?: string;