newo 3.6.2 → 3.7.1

package/src/cli/commands/check.ts

@@ -0,0 +1,21 @@
+/**
+ * `newo check` - umbrella command equivalent to running lint + format --check.
+ *
+ * A failing check exits non-zero if any of the sub-checks fail, so CI
+ * pipelines can gate merges on a single invocation.
+ */
+import { handleLintCommand } from './lint.js';
+import { handleFormatCommand } from './format.js';
+import type { MultiCustomerConfig, CliArgs } from '../../types.js';
+
+export async function handleCheckCommand(
+  customerConfig: MultiCustomerConfig,
+  args: CliArgs,
+  verbose: boolean,
+): Promise<void> {
+  const lintArgs = { ...args };
+  await handleLintCommand(customerConfig, lintArgs as CliArgs, verbose);
+
+  const formatCheckArgs = { ...args, check: true } as CliArgs;
+  await handleFormatCommand(customerConfig, formatCheckArgs, verbose);
+}

package/src/cli/commands/format.ts

@@ -0,0 +1,131 @@
+/**
+ * `newo format` - apply canonical formatting to DSL files.
+ *
+ * Invokes newo-dsl-analyzer's Formatter. In v1 the formatter is an
+ * identity transform (just ensures a final newline). Concrete rules
+ * land in subsequent versions; the command surface is stable now so
+ * CI pipelines and pre-commit hooks can wire `newo format --check`
+ * immediately.
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import { createFormatter } from 'newo-dsl-analyzer';
+
+import { selectSingleCustomer } from '../customer-selection.js';
+import { handleCliError } from '../errors.js';
+import { resolveFormat } from '../../format/detect.js';
+import { discoverCustomerFiles, discoverFromPath, defaultRoot } from '../../lint/discovery.js';
+import type { MultiCustomerConfig, CliArgs, CustomerConfig } from '../../types.js';
+import type { FormatVersion } from '../../format/types.js';
+
+interface FormatArgs {
+  positional: string[];
+  formatVersion: string | undefined;
+  check: boolean;
+  customer: string | undefined;
+}
+
+export async function handleFormatCommand(
+  customerConfig: MultiCustomerConfig,
+  args: CliArgs,
+  verbose: boolean,
+): Promise<void> {
+  try {
+    const fmtArgs = parseArgs(args);
+    const formatter = createFormatter();
+
+    const hasCustomerContext =
+      fmtArgs.customer !== undefined ||
+      Object.keys(customerConfig.customers ?? {}).length > 0;
+
+    const { selectedCustomer, allCustomers, isMultiCustomer } = hasCustomerContext
+      ? selectSingleCustomer(customerConfig, fmtArgs.customer)
+      : { selectedCustomer: null, allCustomers: [] as CustomerConfig[], isMultiCustomer: false };
+
+    const targetCustomer = selectedCustomer ?? (isMultiCustomer ? null : allCustomers[0] ?? null);
+    void targetCustomer;
+
+    const files = await resolveFiles(targetCustomer, allCustomers, fmtArgs, isMultiCustomer);
+    if (files.length === 0) {
+      console.log('No files matched.');
+      return;
+    }
+
+    let touched = 0;
+    let needsFormat = 0;
+    for (const file of files) {
+      const source = await fs.readFile(file.absPath, 'utf8');
+      const result = formatter.format(source, file.absPath);
+      if (!result.changed) continue;
+      needsFormat++;
+      if (fmtArgs.check) {
+        console.log(`would rewrite ${path.relative(process.cwd(), file.absPath)}`);
+      } else {
+        await fs.writeFile(file.absPath, result.formatted, 'utf8');
+        touched++;
+        if (verbose) console.log(`formatted ${path.relative(process.cwd(), file.absPath)}`);
+      }
+    }
+
+    if (fmtArgs.check) {
+      if (needsFormat === 0) {
+        console.log('All files are properly formatted.');
+        return;
+      }
+      console.log(`${needsFormat} file(s) would be reformatted.`);
+      process.exit(1);
+    } else {
+      console.log(`Formatted ${touched} file(s).`);
+    }
+  } catch (err) {
+    handleCliError(err, 'format');
+  }
+}
+
+function parseArgs(args: CliArgs): FormatArgs {
+  const positional = args._.slice(1).filter((p): p is string => typeof p === 'string');
+  return {
+    positional,
+    formatVersion: args.format as string | undefined,
+    check: Boolean(args.check),
+    customer: args.customer as string | undefined,
+  };
+}
+
+async function resolveFiles(
+  selected: CustomerConfig | null,
+  all: CustomerConfig[],
+  args: FormatArgs,
+  isMultiCustomer: boolean,
+) {
+  if (args.positional.length > 0) {
+    const files = [];
+    for (const p of args.positional) {
+      files.push(
+        ...(await discoverFromPath(p, {
+          ...(args.formatVersion ? { format: toFormatVersion(args.formatVersion) } : {}),
+        })),
+      );
+    }
+    return files;
+  }
+  if (selected) {
+    const formatVersion = resolveFormat(selected.idn, args.formatVersion).version;
+    return discoverCustomerFiles(selected, { format: formatVersion });
+  }
+  if (isMultiCustomer) {
+    const files = [];
+    for (const customer of all) {
+      const formatVersion = resolveFormat(customer.idn, args.formatVersion).version;
+      files.push(...(await discoverCustomerFiles(customer, { format: formatVersion })));
+    }
+    return files;
+  }
+  return discoverFromPath(defaultRoot(), {
+    ...(args.formatVersion ? { format: toFormatVersion(args.formatVersion) } : {}),
+  });
+}
+
+function toFormatVersion(v: string): FormatVersion {
+  return v === 'newo_v2' ? 'newo_v2' : 'cli_v1';
+}

package/src/cli/commands/help.ts

@@ -22,6 +22,19 @@ Core Commands:
   newo meta [--customer <idn>]                  # get project metadata (debug)
   newo import-akb <file> <persona_id> [--customer <idn>]  # import AKB articles
 
+Linting & Formatting (NEW):
+  newo lint [paths...] [--format <fmt>] [--reporter <text|json|sarif>]       # static-analysis on DSL files
+  newo lint --changed                                                        # lint only files modified since last push
+  newo lint --live                                                           # refresh action catalog from NEWO API
+  newo lint --rule <code> --no-rule <code>                                   # selectively enable/disable rules
+  newo lint --max-warnings <n>                                               # fail when warnings exceed threshold
+  newo format [paths...] [--check]                                           # apply canonical formatting (in-place or --check)
+  newo check [paths...]                                                      # umbrella: lint + format --check (CI gate)
+
+  Powered by newo-dsl-analyzer. Exit codes: 0 clean, 1 lint errors, 2 runtime error.
+  Discover rules via the Diagnostic code column (E100, W101, ...); configure
+  them in .neworc.yaml at your repo root. Plugin authors: depend on newo-dsl-core.
+
 Project Management:
   newo create-project <idn> [--title <title>] [--description <desc>] [--version <version>] [--auto-update]  # create empty project on platform
   newo list-registries [--customer <idn>]                                   # list available project registries (production, staging, etc.)

package/src/cli/commands/lint.ts

@@ -0,0 +1,246 @@
+/**
+ * `newo lint` - static analysis over Guidance / Jinja / NSL / NSLG files.
+ *
+ * Wraps newo-dsl-analyzer with newo-cli's customer/format/hash primitives.
+ * Exit codes:
+ *   0  clean (or only warnings below --max-warnings)
+ *   1  lint errors found, or warning threshold exceeded
+ *   2  unexpected runtime failure (handled by handleCliError)
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import {
+  createLinter,
+  type LinterOptions,
+  type ProjectLintReport,
+  type RuleSeverity,
+} from 'newo-dsl-analyzer';
+
+import { selectSingleCustomer } from '../customer-selection.js';
+import { handleCliError } from '../errors.js';
+import { resolveFormat } from '../../format/detect.js';
+import { discoverCustomerFiles, discoverFromPath, defaultRoot } from '../../lint/discovery.js';
+import { loadNewoLintConfig } from '../../lint/config.js';
+import { refreshLiveSchema, loadCachedLiveSchema } from '../../lint/live-schema.js';
+import { pickReporter } from '../../lint/reporters/index.js';
+import type { MultiCustomerConfig, CliArgs, CustomerConfig } from '../../types.js';
+
+interface LintArgs {
+  positional: string[];
+  formatVersion: string | undefined;
+  reporter: string;
+  maxWarnings: number;
+  quiet: boolean;
+  rules: string[];
+  noRules: string[];
+  changed: boolean;
+  live: boolean;
+  customer: string | undefined;
+}
+
+export async function handleLintCommand(
+  customerConfig: MultiCustomerConfig,
+  args: CliArgs,
+  verbose: boolean,
+): Promise<void> {
+  try {
+    const lintArgs = parseArgs(args);
+    const report = await run(customerConfig, lintArgs, verbose);
+    const reporter = pickReporter(lintArgs.reporter);
+    const output = reporter.write(report);
+    if (output.trim().length > 0) process.stdout.write(output + '\n');
+
+    const exitCode = determineExitCode(report, lintArgs);
+    if (exitCode !== 0) process.exit(exitCode);
+  } catch (err) {
+    handleCliError(err, 'lint');
+  }
+}
+
+function parseArgs(args: CliArgs): LintArgs {
+  const positional = args._.slice(1).filter((p): p is string => typeof p === 'string');
+
+  // Multiple flag shapes: --rule=E100, --rule E100,W100
+  // Disabling rules uses --rule-off (not --no-rule) because minimist treats
+  // `--no-X` as `X: false` and swallows the next positional argument.
+  const rules = collectCsv(args.rule);
+  const noRules = collectCsv(args['rule-off']);
+
+  return {
+    positional,
+    formatVersion: args.format as string | undefined,
+    reporter: (args.reporter as string | undefined) ?? (args['output-format'] as string | undefined) ?? 'text',
+    maxWarnings: parseIntOr(args['max-warnings'], Number.POSITIVE_INFINITY),
+    quiet: Boolean(args.quiet),
+    rules,
+    noRules,
+    changed: Boolean(args.changed),
+    live: Boolean(args.live),
+    customer: args.customer as string | undefined,
+  };
+}
+
+async function run(
+  customerConfig: MultiCustomerConfig,
+  args: LintArgs,
+  verbose: boolean,
+): Promise<ProjectLintReport> {
+  const config = loadNewoLintConfig();
+
+  // With explicit positional paths AND no customer/live flag, skip customer
+  // selection entirely - lint operates purely on the given filesystem paths.
+  const hasCustomerContext =
+    args.customer !== undefined ||
+    args.live ||
+    Object.keys(customerConfig.customers ?? {}).length > 0;
+
+  const { selectedCustomer, allCustomers, isMultiCustomer } = hasCustomerContext
+    ? selectSingleCustomer(customerConfig, args.customer)
+    : { selectedCustomer: null, allCustomers: [] as CustomerConfig[], isMultiCustomer: false };
+
+  const targetCustomer = selectedCustomer ?? (isMultiCustomer ? null : allCustomers[0] ?? null);
+
+  const schemas = await resolveSchemas(targetCustomer, args, verbose);
+
+  const ruleOverrides: Record<string, RuleSeverity> = {
+    ...(config.rules ?? {}),
+  };
+  for (const code of args.noRules) ruleOverrides[code] = 'off';
+  // --rule enables; we map unknown codes to 'warning' to avoid silently accepting typos.
+  for (const code of args.rules) {
+    if (!(code in ruleOverrides) || ruleOverrides[code] === 'off') {
+      ruleOverrides[code] = 'warning';
+    }
+  }
+
+  const linterOpts: LinterOptions = {
+    rules: ruleOverrides,
+    ...(schemas !== undefined ? { schemas } : {}),
+  };
+  const linter = createLinter(linterOpts);
+
+  const files = await resolveFiles(targetCustomer, allCustomers, args, isMultiCustomer);
+  if (files.length === 0) {
+    return { results: [], errorCount: 0, warningCount: 0 };
+  }
+
+  let errorCount = 0;
+  let warningCount = 0;
+  const results = [];
+  for (const file of files) {
+    const source = await fs.readFile(file.absPath, 'utf8');
+    const result = linter.lint(source, file.absPath);
+    for (const d of result.diagnostics) {
+      if (d.severity === 'error') errorCount++;
+      else if (d.severity === 'warning') warningCount++;
+    }
+    if (args.quiet) {
+      result.diagnostics = result.diagnostics.filter(d => d.severity === 'error');
+    }
+    results.push(result);
+  }
+  return { results, errorCount, warningCount: args.quiet ? 0 : warningCount };
+}
+
+async function resolveSchemas(
+  customer: CustomerConfig | null,
+  args: LintArgs,
+  verbose: boolean,
+): Promise<LinterOptions['schemas']> {
+  if (!customer) return 'bundled';
+
+  if (args.live) {
+    if (verbose) console.log(`Refreshing live schemas for ${customer.idn}...`);
+    const snapshot = await refreshLiveSchema(customer);
+    return { kind: 'inline', actions: snapshot.actions };
+  }
+
+  // Auto-use cached live snapshot if it exists (faster, always specific to
+  // the customer's actual NEWO account state). Fall back to bundled.
+  const cached = await loadCachedLiveSchema(customer.idn);
+  if (cached) {
+    if (verbose) {
+      const age = Math.round((Date.now() - Date.parse(cached.fetchedAt)) / 1000 / 60);
+      console.log(`Using cached schemas for ${customer.idn} (${age} min old). Use --live to refresh.`);
+    }
+    return { kind: 'inline', actions: cached.actions };
+  }
+
+  return 'bundled';
+}
+
+async function resolveFiles(
+  selected: CustomerConfig | null,
+  all: CustomerConfig[],
+  args: LintArgs,
+  isMultiCustomer: boolean,
+): ReturnType<typeof discoverFromPath> {
+  // Explicit positional paths beat everything else.
+  if (args.positional.length > 0) {
+    const files = [];
+    for (const p of args.positional) {
+      const discovered = await discoverFromPath(p, {
+        ...(args.formatVersion ? { format: toFormatVersion(args.formatVersion) } : {}),
+      });
+      files.push(...discovered);
+    }
+    return files;
+  }
+
+  if (selected) {
+    const formatVersion = resolveFormat(selected.idn, args.formatVersion).version;
+    return discoverCustomerFiles(selected, {
+      format: formatVersion,
+      changedOnly: args.changed,
+    });
+  }
+
+  if (isMultiCustomer) {
+    const files = [];
+    for (const customer of all) {
+      const formatVersion = resolveFormat(customer.idn, args.formatVersion).version;
+      const customerFiles = await discoverCustomerFiles(customer, {
+        format: formatVersion,
+        changedOnly: args.changed,
+      });
+      files.push(...customerFiles);
+    }
+    return files;
+  }
+
+  // No customer context - lint cwd / newo_customers/ directly.
+  return discoverFromPath(defaultRoot(), {
+    ...(args.formatVersion ? { format: toFormatVersion(args.formatVersion) } : {}),
+  });
+}
+
+function determineExitCode(report: ProjectLintReport, args: LintArgs): number {
+  if (report.errorCount > 0) return 1;
+  if (report.warningCount > args.maxWarnings) return 1;
+  return 0;
+}
+
+function collectCsv(value: unknown): string[] {
+  if (value === undefined || value === null) return [];
+  const items = Array.isArray(value) ? value : [value];
+  return items
+    .flatMap((v: unknown) => String(v).split(','))
+    .map(s => s.trim())
+    .filter(s => s.length > 0);
+}
+
+function parseIntOr(value: unknown, fallback: number): number {
+  if (typeof value === 'number' && Number.isFinite(value)) return value;
+  if (typeof value === 'string') {
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isNaN(parsed)) return parsed;
+  }
+  return fallback;
+}
+
+function toFormatVersion(v: string): 'cli_v1' | 'newo_v2' {
+  return v === 'newo_v2' ? 'newo_v2' : 'cli_v1';
+}
+
+// Silence unused path import warning; path is used via discovery helpers.
+void path;

package/src/cli.ts

@@ -47,19 +47,14 @@ import { handleWatchCommand } from './cli/commands/watch.js';
 import { handleDiffCommand } from './cli/commands/diff.js';
 import { handleLogsCommand } from './cli/commands/logs.js';
 import { handleExportCommand } from './cli/commands/export.js';
+import { handleLintCommand } from './cli/commands/lint.js';
+import { handleFormatCommand } from './cli/commands/format.js';
+import { handleCheckCommand } from './cli/commands/check.js';
 import type { CliArgs, NewoApiError } from './types.js';
 
 dotenv.config();
 
 async function main(): Promise<void> {
-  try {
-    // Initialize and validate environment at startup
-    initializeEnvironment();
-  } catch (error: unknown) {
-    console.error('Environment validation failed:', error instanceof Error ? error.message : String(error));
-    process.exit(1);
-  }
-
   const args = minimist(process.argv.slice(2)) as CliArgs;
   const cmd = args._[0];
   const verbose = Boolean(args.verbose || args.v);
@@ -72,12 +67,46 @@ async function main(): Promise<void> {
 
   if (verbose) console.log(`🔍 Command parsed: "${cmd}"`);
 
-  // Handle help command first
+  // Handle help command first - no env or customer config needed
   if (!cmd || ['help', '-h', '--help'].includes(cmd)) {
     handleHelpCommand();
     return;
   }
 
+  // Offline commands: lint/format/check don't need NEWO credentials
+  // UNLESS the user passes --customer or --live (both touch the API).
+  const isOfflineLint =
+    (cmd === 'lint' || cmd === 'format' || cmd === 'check') &&
+    !args.customer &&
+    !args.live;
+
+  if (isOfflineLint) {
+    try {
+      const emptyConfig: import('./types.js').MultiCustomerConfig = { customers: {} };
+      switch (cmd) {
+        case 'lint':
+          await handleLintCommand(emptyConfig, args, verbose);
+          return;
+        case 'format':
+          await handleFormatCommand(emptyConfig, args, verbose);
+          return;
+        case 'check':
+          await handleCheckCommand(emptyConfig, args, verbose);
+          return;
+      }
+    } catch (error: unknown) {
+      handleCliError(error, cmd);
+    }
+  }
+
+  // All other commands (and customer-scoped lint/format/check) require env + customer config.
+  try {
+    initializeEnvironment();
+  } catch (error: unknown) {
+    console.error('Environment validation failed:', error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  }
+
   // Handle list-customers command (doesn't need full customer config)
   if (cmd === 'list-customers') {
     try {
@@ -249,6 +278,18 @@ async function main(): Promise<void> {
         await handleDiffCommand(customerConfig, args, verbose);
         break;
 
+      case 'lint':
+        await handleLintCommand(customerConfig, args, verbose);
+        break;
+
+      case 'format':
+        await handleFormatCommand(customerConfig, args, verbose);
+        break;
+
+      case 'check':
+        await handleCheckCommand(customerConfig, args, verbose);
+        break;
+
       default:
         console.error('Unknown command:', cmd);
         console.error('Run "newo --help" for usage information');

package/src/domain/strategies/sync/AttributeSyncStrategy.ts

@@ -39,6 +39,11 @@ import {
   customerAttributesMapPath
 } from '../../../fsutil.js';
 import { patchYamlToPyyaml } from '../../../format/yaml-patch.js';
+import {
+  isJsonValueType,
+  normalizeJsonValueForStorage,
+  jsonValuesEqual
+} from '../../../sync/json-attr-utils.js';
 import { sha256, saveHashes, loadHashes } from '../../../hash.js';
 
 /**
@@ -220,8 +225,14 @@ export class AttributeSyncStrategy implements ISyncStrategy<CustomerAttributesRe
   private cleanAttribute(attr: CustomerAttribute): CustomerAttribute {
     let processedValue = attr.value;
 
-    // Handle JSON string values
-    if (typeof attr.value === 'string' && attr.value.startsWith('[{') && attr.value.endsWith('}]')) {
+    // Coerce JSON-typed values to a STRING. The API may return parsed
+    // objects for `value_type: json`; if we let yaml.dump turn them into
+    // YAML structures, the next push sends an object and the Workflow
+    // Builder canvas blanks out. See src/sync/json-attr-utils.ts.
+    if (isJsonValueType(attr.value_type)) {
+      processedValue = normalizeJsonValueForStorage(attr.value);
+    } else if (typeof attr.value === 'string' && attr.value.startsWith('[{') && attr.value.endsWith('}]')) {
+      // Legacy: reformat array-of-objects JSON strings for readability
       try {
         const parsed = JSON.parse(attr.value);
         processedValue = JSON.stringify(parsed, null, 0);
@@ -342,10 +353,24 @@ export class AttributeSyncStrategy implements ISyncStrategy<CustomerAttributesRe
       const remoteAttr = remoteMap.get(localAttr.idn);
       if (!remoteAttr) continue;
 
-      if (String(localAttr.value) !== String(remoteAttr.value)) {
+      // For JSON-typed attrs, compare canonical JSON (handles
+      // pretty/compact and string/object differences). Always send the
+      // value as a STRING so the platform stores the canvas the way the
+      // Workflow Builder expects to read it back.
+      const isJson = isJsonValueType(localAttr.value_type);
+      const valuesAreEqual = isJson
+        ? jsonValuesEqual(localAttr.value, remoteAttr.value)
+        : String(localAttr.value) === String(remoteAttr.value);
+
+      if (!valuesAreEqual) {
+        const valueToSend = isJson
+          ? normalizeJsonValueForStorage(localAttr.value)
+          : localAttr.value;
+
         await updateCustomerAttribute(client, {
-          id: attributeId,
-          ...localAttr
+          ...localAttr,
+          value: valueToSend,
+          id: attributeId
         });
         updatedCount++;
         this.logger.info(`  ✓ Updated customer attribute: ${localAttr.idn}`);
@@ -399,10 +424,22 @@ export class AttributeSyncStrategy implements ISyncStrategy<CustomerAttributesRe
       const remoteAttr = remoteMap.get(localAttr.idn);
       if (!remoteAttr) continue;
 
-      if (String(localAttr.value) !== String(remoteAttr.value)) {
+      // Same canonical-JSON / always-string-on-push policy as customer
+      // attributes (see pushCustomerAttributes for rationale).
+      const isJson = isJsonValueType(localAttr.value_type);
+      const valuesAreEqual = isJson
+        ? jsonValuesEqual(localAttr.value, remoteAttr.value)
+        : String(localAttr.value) === String(remoteAttr.value);
+
+      if (!valuesAreEqual) {
+        const valueToSend = isJson
+          ? normalizeJsonValueForStorage(localAttr.value)
+          : localAttr.value;
+
         await updateProjectAttribute(client, project.id, {
-          id: attributeId,
-          ...localAttr
+          ...localAttr,
+          value: valueToSend,
+          id: attributeId
         });
         updatedCount++;
         this.logger.info(`  ✓ Updated project attribute: ${projectIdn}/${localAttr.idn}`);

package/src/lint/config.ts

@@ -0,0 +1,17 @@
+/**
+ * Lint config resolution for the newo CLI.
+ *
+ * Looks for `.neworc.yaml` / `.neworc.yml` / `.neworc.json` starting at
+ * the cwd and walking up to the filesystem root. Thin wrapper around
+ * newo-dsl-analyzer's `loadConfig` so consumers can override location
+ * per command if they need to.
+ */
+import path from 'path';
+import { loadConfig as analyzerLoadConfig } from 'newo-dsl-analyzer';
+import type { NewoLintConfig } from 'newo-dsl-analyzer';
+
+export type { NewoLintConfig } from 'newo-dsl-analyzer';
+
+export function loadNewoLintConfig(startDir: string = process.cwd()): NewoLintConfig {
+  return analyzerLoadConfig(path.resolve(startDir)) ?? {};
+}