newo 3.4.2 → 3.6.0

package/src/format/v2-yaml.ts

@@ -0,0 +1,345 @@
+/**
+ * V2 YAML parsers and generators
+ *
+ * Handles reading/writing the newo_v2 format YAML files:
+ * - Flow YAML: {FlowIdn}.yaml (inline skill definitions, events, state_fields)
+ * - Project YAML: {project_idn}.yaml
+ * - Agent YAML: agent.yaml
+ * - Library YAML: {library_idn}.yaml
+ */
+import fs from 'fs-extra';
+import yaml from 'js-yaml';
+import { patchYamlToPyyaml } from './yaml-patch.js';
+
+// ── V2 Types ──
+
+export interface V2InlineSkill {
+  title: string;
+  idn: string;
+  prompt_script: string;
+  runner_type: string;
+  model: {
+    model_idn: string;
+    provider_idn: string;
+  };
+  parameters: Array<{ name: string; default_value: string }>;
+}
+
+export interface V2FlowEvent {
+  idn: string;
+  skill_selector: string;
+  skill_idn: string | null;
+  state_idn: string | null;
+  integration_idn: string | null;
+  connector_idn: string | null;
+  interrupt_mode: string;
+}
+
+export interface V2StateField {
+  title: string;
+  idn: string;
+  default_value: string;
+  scope: string;
+}
+
+export interface V2FlowDefinition {
+  title: string;
+  idn: string;
+  description: string | null;
+  agent_id: string | null;
+  skills: V2InlineSkill[];
+  events: V2FlowEvent[];
+  state_fields: V2StateField[];
+  default_runner_type: string;
+  default_provider_idn: string;
+  default_model_idn: string;
+  publication_type: string | null;
+}
+
+export interface V2ProjectMeta {
+  idn: string;
+  name: string;
+  version: string;
+  description: string;
+  is_auto_update_enabled: boolean;
+  registry: string;
+  registry_item_idn: string;
+}
+
+export interface V2AgentMeta {
+  idn: string;
+  title: string | null;
+  description: string | null;
+}
+
+export interface V2LibraryDefinition {
+  title: string;
+  idn: string;
+  description: string | null;
+  skills: V2InlineSkill[];
+}
+
+// ── Custom YAML tag for !enum values ──
+
+const enumTag = new yaml.Type('!enum', {
+  kind: 'scalar',
+  resolve: () => true,
+  construct: (data: string) => data,
+  represent: (data: unknown) => String(data),
+});
+
+const V2_YAML_SCHEMA = yaml.DEFAULT_SCHEMA.extend([enumTag]);
+
+// ── Shared YAML dump options ──
+
+const YAML_DUMP_OPTIONS: yaml.DumpOptions = {
+  indent: 2,
+  quotingType: '"',
+  forceQuotes: false,
+  lineWidth: -1,
+  noRefs: true,
+  sortKeys: false,
+  flowLevel: -1,
+  schema: V2_YAML_SCHEMA,
+};
+
+// ── Skill sorting: CamelCase first, then _prefixed, then snake_case ──
+
+function skillSortKey(idn: string): string {
+  if (idn.startsWith('_')) {
+    return `1_${idn}`; // _prefixed second
+  }
+  if (idn[0] && idn[0] === idn[0].toUpperCase()) {
+    return `0_${idn}`; // CamelCase first
+  }
+  return `2_${idn}`; // snake_case last
+}
+
+/**
+ * Sort skills in V2 export order (case-sensitive ASCII sort within groups):
+ * 1. CamelCase (public) - case-sensitive alphabetically
+ * 2. _prefixed (private) - case-sensitive alphabetically
+ * 3. snake_case - case-sensitive alphabetically
+ */
+export function sortV2Skills<T extends { idn: string }>(skills: T[]): T[] {
+  return [...skills].sort((a, b) => {
+    const ka = skillSortKey(a.idn);
+    const kb = skillSortKey(b.idn);
+    return ka < kb ? -1 : ka > kb ? 1 : 0;
+  });
+}
+
+/**
+ * Sort parameters alphabetically by name (case-sensitive, V2 export order)
+ */
+export function sortV2Parameters<T extends { name: string }>(params: T[]): T[] {
+  return [...params].sort((a, b) => {
+    return a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
+  });
+}
+
+// ── Flow YAML ──
+
+export async function parseV2FlowYaml(filePath: string): Promise<V2FlowDefinition> {
+  const content = await fs.readFile(filePath, 'utf8');
+  return yaml.load(content, { schema: V2_YAML_SCHEMA }) as V2FlowDefinition;
+}
+
+/**
+ * Generate V2 flow YAML content from API data
+ *
+ * Produces the exact format found in the reference V2 export:
+ *   title, idn, description, agent_id, skills[], events[], state_fields[],
+ *   default_runner_type, default_provider_idn, default_model_idn, publication_type
+ */
+export function generateV2FlowYaml(
+  flowIdn: string,
+  flowTitle: string,
+  flowDescription: string | null,
+  defaultRunnerType: string,
+  defaultProviderIdn: string,
+  defaultModelIdn: string,
+  skills: V2InlineSkill[],
+  events: V2FlowEvent[],
+  stateFields: V2StateField[]
+): string {
+  // Sort skills in V2 order and sort parameters within each skill
+  const sortedSkills = sortV2Skills(skills).map(s => ({
+    ...s,
+    parameters: sortV2Parameters(s.parameters),
+  }));
+
+  // Sort events by idn, then skill_idn, then integration_idn, then connector_idn
+  const sortedEvents = [...events].sort((a, b) => {
+    if (a.idn !== b.idn) return a.idn < b.idn ? -1 : 1;
+    const as = a.skill_idn || '';
+    const bs = b.skill_idn || '';
+    if (as !== bs) return as < bs ? -1 : 1;
+    const ai = a.integration_idn || '';
+    const bi = b.integration_idn || '';
+    if (ai !== bi) return ai < bi ? -1 : 1;
+    const ac = a.connector_idn || '';
+    const bc = b.connector_idn || '';
+    return ac < bc ? -1 : ac > bc ? 1 : 0;
+  });
+  // Sort state_fields alphabetically by idn
+  const sortedStates = [...stateFields].sort((a, b) => a.idn < b.idn ? -1 : a.idn > b.idn ? 1 : 0);
+
+  const flowDef: V2FlowDefinition = {
+    title: flowTitle,
+    idn: flowIdn,
+    description: flowDescription ?? null,
+    agent_id: null,
+    skills: sortedSkills,
+    events: sortedEvents,
+    state_fields: sortedStates,
+    default_runner_type: defaultRunnerType,
+    default_provider_idn: defaultProviderIdn,
+    default_model_idn: defaultModelIdn,
+    publication_type: null,
+  };
+
+  // Flow YAML uses lineWidth: -1 (no wrapping) to keep prompt_script paths on one line
+  // Then patch to convert double-quoted JSON values to single-quoted
+  return patchYamlToPyyaml(yaml.dump(flowDef, { ...YAML_DUMP_OPTIONS, lineWidth: -1 }));
+}
+
+// ── Project YAML ──
+
+export async function parseV2ProjectYaml(filePath: string): Promise<V2ProjectMeta> {
+  const content = await fs.readFile(filePath, 'utf8');
+  const parsed = yaml.load(content, { schema: V2_YAML_SCHEMA }) as { project: V2ProjectMeta };
+  return parsed.project;
+}
+
+/**
+ * Generate V2 project YAML
+ *
+ * Format:
+ *   project:
+ *     idn: naf
+ *     name: naf
+ *     version: 4.1.0
+ *     description: ""
+ *     is_auto_update_enabled: true
+ *     registry: production
+ *     registry_item_idn: naf
+ */
+export function generateV2ProjectYaml(meta: V2ProjectMeta): string {
+  return yaml.dump({ project: meta }, YAML_DUMP_OPTIONS);
+}
+
+// ── Agent YAML ──
+
+export async function parseV2AgentYaml(filePath: string): Promise<V2AgentMeta> {
+  const content = await fs.readFile(filePath, 'utf8');
+  const parsed = yaml.load(content, { schema: V2_YAML_SCHEMA }) as { agent: V2AgentMeta };
+  return parsed.agent;
+}
+
+/**
+ * Generate V2 agent YAML
+ *
+ * Format:
+ *   agent:
+ *     idn: TaskManager
+ *     title: TaskManager
+ *     description: null
+ *
+ * V2 export preserves description exactly as provided (null stays null, "" stays "")
+ */
+export function generateV2AgentYaml(meta: V2AgentMeta): string {
+  return patchYamlToPyyaml(yaml.dump({ agent: meta }, YAML_DUMP_OPTIONS));
+}
+
+// ── Library YAML ──
+
+export async function parseV2LibraryYaml(filePath: string): Promise<V2LibraryDefinition> {
+  const content = await fs.readFile(filePath, 'utf8');
+  return yaml.load(content, { schema: V2_YAML_SCHEMA }) as V2LibraryDefinition;
+}
+
+/**
+ * Generate V2 library YAML
+ *
+ * Format:
+ *   title: Test Library
+ *   idn: testLib
+ *   description: Shared utility library
+ *   skills:
+ *     - idn: utilSkill
+ *       ...
+ */
+export function generateV2LibraryYaml(lib: V2LibraryDefinition): string {
+  return yaml.dump(lib, YAML_DUMP_OPTIONS);
+}
+
+// ── Conversion helpers ──
+
+/**
+ * Build a V2InlineSkill entry from API skill data
+ */
+export function buildV2InlineSkill(
+  skillIdn: string,
+  skillTitle: string,
+  runnerType: string,
+  modelIdn: string,
+  providerIdn: string,
+  parameters: Array<{ name: string; default_value: string }>,
+  promptScriptRelPath: string
+): V2InlineSkill {
+  return {
+    title: skillTitle || '',
+    idn: skillIdn,
+    prompt_script: promptScriptRelPath,
+    runner_type: runnerType,
+    model: {
+      model_idn: modelIdn,
+      provider_idn: providerIdn,
+    },
+    parameters: parameters.map(p => ({
+      name: p.name,
+      default_value: p.default_value ?? '',
+    })),
+  };
+}
+
+/**
+ * Build a V2FlowEvent entry from API event data
+ */
+export function buildV2FlowEvent(
+  eventIdn: string,
+  skillSelector: string,
+  skillIdn: string | null,
+  stateIdn: string | null,
+  integrationIdn: string | null,
+  connectorIdn: string | null,
+  interruptMode: string
+): V2FlowEvent {
+  return {
+    idn: eventIdn,
+    skill_selector: skillSelector,
+    skill_idn: skillIdn || null,
+    state_idn: stateIdn || null,
+    integration_idn: integrationIdn || null,
+    connector_idn: connectorIdn || null,
+    interrupt_mode: interruptMode,
+  };
+}
+
+/**
+ * Build a V2StateField entry from API state data
+ */
+export function buildV2StateField(
+  stateIdn: string,
+  stateTitle: string,
+  defaultValue: string,
+  scope: string
+): V2StateField {
+  return {
+    title: stateTitle || '',
+    idn: stateIdn,
+    default_value: defaultValue ?? '',
+    scope,
+  };
+}

package/src/format/yaml-patch.ts

@@ -0,0 +1,208 @@
+/**
+ * YAML Post-Processor - Patches js-yaml output to match pyyaml's formatting
+ *
+ * Replicates pyyaml's Emitter wrapping behavior:
+ * - Double-quoted: breaks with `\` when column + pending > best_width (80) at space
+ * - Plain scalar: breaks at space when column > best_width
+ * - Continuation indent: parent indent + best_indent (usually +2)
+ * - Single-quote preference for strings with brackets
+ */
+
+const BEST_WIDTH = 80;
+
+/**
+ * Patch full YAML document output to match pyyaml formatting
+ */
+export function patchYamlToPyyaml(yamlText: string): string {
+  const lines = yamlText.split('\n');
+  const result: string[] = [];
+  let inBlockScalar = false;
+  let blockIndent = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]!;
+
+    // Track block scalar context (|- or >- or | or >)
+    if (inBlockScalar) {
+      const currentIndent = line.length - line.trimStart().length;
+      if (line.trim() === '' || currentIndent > blockIndent) {
+        result.push(line);
+        continue;
+      }
+      inBlockScalar = false;
+    }
+
+    // Detect start of block scalar
+    if (/^\s*[\w-]+:\s+[|>]-?\s*$/.test(line)) {
+      const keyIndent = line.length - line.trimStart().length;
+      inBlockScalar = true;
+      blockIndent = keyIndent;
+      result.push(line);
+      continue;
+    }
+
+    result.push(...patchLine(line));
+  }
+
+  return result.join('\n');
+}
+
+// Keys that must NEVER be wrapped
+const NO_WRAP_KEYS = new Set([
+  'prompt_script', 'idn', 'runner_type',
+  'model_idn', 'provider_idn', 'skill_idn', 'state_idn',
+  'integration_idn', 'connector_idn', 'interrupt_mode',
+  'skill_selector', 'name', 'scope', 'agent_id',
+  'default_runner_type', 'default_provider_idn', 'default_model_idn',
+  'publication_type', 'is_hidden', 'is_auto_update_enabled',
+  'group', 'registry', 'registry_item_idn', 'version',
+]);
+
+function patchLine(line: string): string[] {
+  if (line.trim() === '' || line.trim().startsWith('#')) {
+    return [line];
+  }
+
+  const kvMatch = line.match(/^(\s*(?:-\s+)?)([\w-]+):\s+(.+)$/);
+  if (!kvMatch) {
+    return [line];
+  }
+
+  const prefix = kvMatch[1]!;
+  const key = kvMatch[2]!;
+  const value = kvMatch[3]!;
+
+  // Single-quote fix for JSON-like values (before anything else)
+  const sqFix = tryConvertToSingleQuote(value);
+  const effectiveValue = sqFix ?? value;
+  const effectiveLine = sqFix !== null ? `${prefix}${key}: ${sqFix}` : line;
+
+  if (NO_WRAP_KEYS.has(key)) {
+    return [effectiveLine];
+  }
+
+  // Only wrap if line exceeds BEST_WIDTH
+  if (effectiveLine.length <= BEST_WIDTH) {
+    return [effectiveLine];
+  }
+
+  const keyPart = `${prefix}${key}: `;
+  // pyyaml continuation indent = current mapping indent + best_indent
+  // For "  description: ..." indent is 2, continuation = 2 + 2 = 4 spaces
+  const keyIndent = prefix.replace(/-\s+$/, '').length;
+  const contIndent = ' '.repeat(keyIndent + 2);
+
+  if (effectiveValue.startsWith('"') && effectiveValue.endsWith('"')) {
+    return wrapDoubleQuoted(keyPart, effectiveValue, contIndent);
+  }
+
+  if (effectiveValue.startsWith("'") && effectiveValue.endsWith("'")) {
+    return [effectiveLine]; // Single-quoted: don't wrap
+  }
+
+  return wrapPlainScalar(keyPart, effectiveValue, contIndent);
+}
+
+/**
+ * Try to convert double-quoted string with escaped chars to single-quoted
+ */
+function tryConvertToSingleQuote(value: string): string | null {
+  if (!value.startsWith('"') || !value.endsWith('"')) return null;
+
+  const inner = value.slice(1, -1);
+  if (!inner.includes('\\"')) return null;
+  if (!inner.includes('[') && !inner.includes('{')) return null;
+
+  const unescaped = inner.replace(/\\"/g, '"').replace(/\\\\/g, '\\');
+
+  if (unescaped.includes("'")) {
+    return `'${unescaped.replace(/'/g, "''")}'`;
+  }
+  return `'${unescaped}'`;
+}
+
+/**
+ * Wrap double-quoted scalar matching pyyaml's write_double_quoted algorithm:
+ * - Track column from 0
+ * - At each space, check if column + pending > best_width
+ * - If yes, emit text + `\`, newline, indent, `\ ` (escaped space for continuation)
+ */
+function wrapDoubleQuoted(keyPart: string, quotedValue: string, contIndent: string): string[] {
+  const inner = quotedValue.slice(1, -1);
+  const result: string[] = [];
+
+  let column = keyPart.length + 1; // keyPart + opening "
+  let lineStart = 0;
+  let lastSpace = -1;
+
+  for (let i = 0; i < inner.length; i++) {
+    column++;
+    if (inner[i] === ' ') {
+      lastSpace = i;
+    }
+
+    // pyyaml condition: column + remaining_in_word > best_width, at a space or start >= end
+    if (column > BEST_WIDTH && lastSpace > lineStart) {
+      // Break at lastSpace
+      const chunk = inner.slice(lineStart, lastSpace);
+      if (result.length === 0) {
+        result.push(`${keyPart}"${chunk}\\`);
+      } else {
+        result.push(`${contIndent}\\ ${chunk}\\`);
+      }
+      lineStart = lastSpace + 1; // skip the space
+      column = contIndent.length + 2 + (i - lastSpace); // contIndent + "\ " + chars after space
+      lastSpace = -1;
+    }
+  }
+
+  // Remaining text
+  const remaining = inner.slice(lineStart);
+  if (result.length === 0) {
+    result.push(`${keyPart}"${remaining}"`);
+  } else {
+    result.push(`${contIndent}\\ ${remaining}"`);
+  }
+
+  return result;
+}
+
+/**
+ * Wrap plain scalar matching pyyaml's write_plain algorithm:
+ * - At each space, if column > best_width, break
+ * - Continuation is just indented text (no backslash)
+ */
+function wrapPlainScalar(keyPart: string, value: string, contIndent: string): string[] {
+  const result: string[] = [];
+  let column = keyPart.length;
+  let lineStart = 0;
+  let lastSpace = -1;
+
+  for (let i = 0; i < value.length; i++) {
+    column++;
+    if (value[i] === ' ') {
+      // pyyaml breaks at single space when column > best_width
+      if (column > BEST_WIDTH && lastSpace >= lineStart) {
+        const chunk = value.slice(lineStart, i);
+        if (result.length === 0) {
+          result.push(`${keyPart}${chunk}`);
+        } else {
+          result.push(`${contIndent}${chunk}`);
+        }
+        lineStart = i + 1; // skip the space
+        column = contIndent.length;
+      }
+      lastSpace = i;
+    }
+  }
+
+  // Remaining
+  const remaining = value.slice(lineStart);
+  if (result.length === 0) {
+    result.push(`${keyPart}${remaining}`);
+  } else {
+    result.push(`${contIndent}${remaining}`);
+  }
+
+  return result;
+}

package/src/fsutil.ts

@@ -31,6 +31,15 @@ export async function ensureState(customerIdn: string): Promise<void> {
   await fs.ensureDir(customerProjectsDir(customerIdn));
 }
 
+/**
+ * Ensure only the .newo/ state directory exists (no V1 projects/ dir)
+ * Used by newo_v2 format to avoid creating V1 artifacts
+ */
+export async function ensureStateOnly(customerIdn: string): Promise<void> {
+  await fs.ensureDir(STATE_DIR);
+  await fs.ensureDir(customerStateDir(customerIdn));
+}
+
 export function projectDir(customerIdn: string, projectIdn: string): string {
   return path.posix.join(customerProjectsDir(customerIdn), projectIdn);
 }
@@ -110,6 +119,34 @@ export function skillMetadataPath(
   return path.posix.join(skillFolderPath(customerIdn, projectIdn, agentIdn, flowIdn, skillIdn), 'metadata.yaml');
 }
 
+// Library paths (cli_v1 format)
+export function libraryDir(customerIdn: string, projectIdn: string, libraryIdn: string): string {
+  return path.posix.join(customerProjectsDir(customerIdn), projectIdn, 'libraries', libraryIdn);
+}
+
+export function libraryMetadataPath(customerIdn: string, projectIdn: string, libraryIdn: string): string {
+  return path.posix.join(libraryDir(customerIdn, projectIdn, libraryIdn), 'metadata.yaml');
+}
+
+export function librarySkillFolderPath(
+  customerIdn: string, projectIdn: string, libraryIdn: string, skillIdn: string
+): string {
+  return path.posix.join(libraryDir(customerIdn, projectIdn, libraryIdn), skillIdn);
+}
+
+export function librarySkillScriptPath(
+  customerIdn: string, projectIdn: string, libraryIdn: string, skillIdn: string, runnerType: RunnerType = 'guidance'
+): string {
+  const extension = runnerType === 'nsl' ? '.jinja' : '.guidance';
+  return path.posix.join(librarySkillFolderPath(customerIdn, projectIdn, libraryIdn, skillIdn), `${skillIdn}${extension}`);
+}
+
+export function librarySkillMetadataPath(
+  customerIdn: string, projectIdn: string, libraryIdn: string, skillIdn: string
+): string {
+  return path.posix.join(librarySkillFolderPath(customerIdn, projectIdn, libraryIdn, skillIdn), 'metadata.yaml');
+}
+
 // Legacy metadata path - keep for backwards compatibility
 export function metadataPath(customerIdn: string, projectIdn: string): string {
   return path.posix.join(customerProjectsDir(customerIdn), projectIdn, 'metadata.json');

package/src/sync/attributes.ts

@@ -298,9 +298,9 @@ export async function pushProjectAttributes(
     // Value type is already parsed (we removed !enum tags above)
     const valueType = localAttr.value_type;
 
-    // Check if value changed
-    const localValue = String(localAttr.value || '');
-    const remoteValue = String(remoteAttr.value || '');
+    // Check if value changed (use ?? to preserve 0, false, empty string)
+    const localValue = String(localAttr.value ?? '');
+    const remoteValue = String(remoteAttr.value ?? '');
 
     if (localValue !== remoteValue) {
       if (verbose) console.log(`   🔄 Updating project attribute: ${localAttr.idn}`);

package/src/sync/skill-files.ts

@@ -69,8 +69,8 @@ export async function findSkillScriptFiles(skillFolderPath: string): Promise<Ski
     if (stats.isFile()) {
       const ext = path.extname(fileName).toLowerCase();
 
-      // Check for script file extensions
-      if (['.jinja', '.guidance', '.nsl'].includes(ext)) {
+      // Check for script file extensions (all formats)
+      if (['.jinja', '.guidance', '.nsl', '.nslg'].includes(ext)) {
         const content = await fs.readFile(filePath, 'utf8');
         scriptFiles.push({
           filePath,

package/src/types.ts

@@ -182,10 +182,16 @@ export interface AgentData {
   flows: Record<string, FlowData>;
 }
 
+export interface LibraryData {
+  id: string;
+  skills: Record<string, SkillMetadata>;
+}
+
 export interface ProjectData {
   projectId: string;
   projectIdn: string;
   agents: Record<string, AgentData>;
+  libraries?: Record<string, LibraryData> | undefined;
 }
 
 export interface ProjectMap {