newo 3.6.2 → 3.7.1

package/src/lint/discovery.ts

@@ -0,0 +1,148 @@
+/**
+ * File discovery for `newo lint` / `newo format` / `newo check`.
+ *
+ * Walks a customer's tree (or any directory passed on the CLI), filters
+ * by format-aware extensions, and optionally narrows to files changed
+ * since the last push by consulting `.newo/{customer}/hashes.json`.
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import { NEWO_CUSTOMERS_DIR, customerDir } from '../fsutil.js';
+import { loadHashes } from '../hash.js';
+import { sha256 } from '../hash.js';
+import {
+  ALL_SCRIPT_EXTENSIONS,
+  type FormatVersion,
+  CLI_V1_EXTENSIONS,
+  NEWO_V2_EXTENSIONS,
+} from '../format/types.js';
+import type { CustomerConfig } from '../types.js';
+
+export interface DiscoveryOptions {
+  /** If set, restrict to one format's extensions only. */
+  format?: FormatVersion;
+  /** If true, include only files that hash-differ from `.newo/{customer}/hashes.json`. */
+  changedOnly?: boolean;
+  /** Additional absolute paths to skip. */
+  ignore?: string[];
+}
+
+export interface DiscoveredFile {
+  /** Absolute path on disk. */
+  absPath: string;
+  /** Path relative to the customer root (for display / reports). */
+  relPath: string;
+  /** Extension (including leading dot). */
+  ext: string;
+}
+
+/**
+ * Discover script files under a customer's tree.
+ * Respects format when given (else walks all recognized extensions).
+ */
+export async function discoverCustomerFiles(
+  customer: CustomerConfig,
+  opts: DiscoveryOptions = {},
+): Promise<DiscoveredFile[]> {
+  const root = customerDir(customer.idn);
+  if (!(await fs.pathExists(root))) return [];
+
+  const exts = pickExtensions(opts.format);
+  const ignoreSet = new Set(opts.ignore ?? []);
+
+  const hits = await walkForExtensions(root, exts, ignoreSet);
+
+  if (!opts.changedOnly) {
+    return hits.map(absPath => toDiscoveredFile(absPath, root));
+  }
+
+  const stored = await loadHashes(customer.idn);
+  const changed: DiscoveredFile[] = [];
+  for (const absPath of hits) {
+    const current = sha256(await fs.readFile(absPath, 'utf8'));
+    if (stored[absPath] !== current) {
+      changed.push(toDiscoveredFile(absPath, root));
+    }
+  }
+  return changed;
+}
+
+/**
+ * Discover files under an arbitrary directory.
+ * Used when the user passes explicit paths to `newo lint some/dir`.
+ */
+export async function discoverFromPath(
+  inputPath: string,
+  opts: DiscoveryOptions = {},
+): Promise<DiscoveredFile[]> {
+  const abs = path.resolve(inputPath);
+  if (!(await fs.pathExists(abs))) return [];
+
+  const exts = pickExtensions(opts.format);
+  const ignoreSet = new Set(opts.ignore ?? []);
+  const stat = await fs.stat(abs);
+
+  let hits: string[];
+  if (stat.isFile()) {
+    hits = exts.includes(path.extname(abs)) ? [abs] : [];
+  } else {
+    hits = await walkForExtensions(abs, exts, ignoreSet);
+  }
+
+  const root = stat.isDirectory() ? abs : path.dirname(abs);
+  return hits.map(p => toDiscoveredFile(p, root));
+}
+
+function pickExtensions(format?: FormatVersion): string[] {
+  if (!format) return [...ALL_SCRIPT_EXTENSIONS];
+  const map = format === 'newo_v2' ? NEWO_V2_EXTENSIONS : CLI_V1_EXTENSIONS;
+  return Object.values(map);
+}
+
+function toDiscoveredFile(absPath: string, root: string): DiscoveredFile {
+  return {
+    absPath,
+    relPath: path.relative(root, absPath),
+    ext: path.extname(absPath),
+  };
+}
+
+async function walkForExtensions(
+  dir: string,
+  exts: string[],
+  ignore: Set<string>,
+): Promise<string[]> {
+  const out: string[] = [];
+  const stack: string[] = [dir];
+  while (stack.length > 0) {
+    const current = stack.pop()!;
+    if (ignore.has(current)) continue;
+    let entries: fs.Dirent[];
+    try {
+      entries = await fs.readdir(current, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      // Skip hidden dirs, node_modules, and the .newo state directory.
+      if (entry.name.startsWith('.')) continue;
+      if (entry.name === 'node_modules') continue;
+      const full = path.join(current, entry.name);
+      if (ignore.has(full)) continue;
+      if (entry.isDirectory()) {
+        stack.push(full);
+      } else if (exts.includes(path.extname(entry.name))) {
+        out.push(full);
+      }
+    }
+  }
+  return out.sort();
+}
+
+/**
+ * Default root for lint invocations with no path arguments - the
+ * `newo_customers/` directory at the cwd.
+ */
+export function defaultRoot(): string {
+  return NEWO_CUSTOMERS_DIR;
+}

package/src/lint/live-schema.ts

@@ -0,0 +1,62 @@
+/**
+ * Live schema refresh: hits `/api/v1/script/actions` via the existing
+ * NEWO api client, caches the response to `.newo/{customer}/actions.json`,
+ * and returns an object shaped for `createLinter({ schemas: { kind: 'inline', ... }})`.
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import { customerStateDir } from '../fsutil.js';
+import type { CustomerConfig, ScriptAction } from '../types.js';
+import { getValidAccessToken } from '../auth.js';
+import { makeClient, getScriptActions } from '../api.js';
+
+export interface LiveSchemaSnapshot {
+  actions: Array<{ name: string; [k: string]: unknown }>;
+  fetchedAt: string;
+}
+
+export async function liveSchemaCachePath(customerIdn: string): Promise<string> {
+  const dir = customerStateDir(customerIdn);
+  await fs.ensureDir(dir);
+  return path.join(dir, 'actions.json');
+}
+
+/**
+ * Fetch the current action catalog from NEWO and cache it.
+ * Returns a snapshot ready to pass to `createLinter`.
+ */
+export async function refreshLiveSchema(
+  customer: CustomerConfig,
+): Promise<LiveSchemaSnapshot> {
+  const token = await getValidAccessToken(customer);
+  const client = await makeClient(false, token);
+  const actions = await getScriptActions(client);
+  const snapshot: LiveSchemaSnapshot = {
+    actions: actions.map((a: ScriptAction) => ({
+      name: a.idn ?? a.title,
+      title: a.title,
+      ...(a.idn !== undefined ? { idn: a.idn } : {}),
+      arguments: a.arguments,
+    })),
+    fetchedAt: new Date().toISOString(),
+  };
+  const cachePath = await liveSchemaCachePath(customer.idn);
+  await fs.writeJson(cachePath, snapshot, { spaces: 2 });
+  return snapshot;
+}
+
+/**
+ * Load the cached snapshot if present. Returns null when the cache is
+ * missing or corrupt (caller should fall back to bundled schemas).
+ */
+export async function loadCachedLiveSchema(
+  customerIdn: string,
+): Promise<LiveSchemaSnapshot | null> {
+  const cachePath = await liveSchemaCachePath(customerIdn);
+  if (!(await fs.pathExists(cachePath))) return null;
+  try {
+    return (await fs.readJson(cachePath)) as LiveSchemaSnapshot;
+  } catch {
+    return null;
+  }
+}

package/src/lint/reporters/index.ts

@@ -0,0 +1,22 @@
+import { textReporter } from './text.js';
+import { jsonReporter } from './json.js';
+import { sarifReporter } from './sarif.js';
+import type { Reporter, ReporterName } from './types.js';
+
+export type { Reporter, ReporterName } from './types.js';
+
+export function pickReporter(name: ReporterName | string | undefined): Reporter {
+  switch (name) {
+    case 'json':
+      return jsonReporter;
+    case 'sarif':
+      return sarifReporter;
+    case 'text':
+    case undefined:
+    case '':
+      return textReporter;
+    default:
+      console.warn(`Unknown --format value '${name}', defaulting to text.`);
+      return textReporter;
+  }
+}

package/src/lint/reporters/json.ts

@@ -0,0 +1,12 @@
+/**
+ * Machine-readable JSON reporter. Matches the ProjectLintReport shape
+ * exactly - consumers can parse with any JSON tool.
+ */
+import type { ProjectLintReport } from 'newo-dsl-analyzer';
+import type { Reporter } from './types.js';
+
+export const jsonReporter: Reporter = {
+  write(report: ProjectLintReport): string {
+    return JSON.stringify(report, null, 2);
+  },
+};

package/src/lint/reporters/sarif.ts

@@ -0,0 +1,59 @@
+/**
+ * SARIF 2.1.0 reporter - lets GitHub Advanced Security / Code Scanning
+ * pick up newo-lint findings alongside CodeQL and other linters.
+ *
+ * Spec: https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html
+ */
+import type { ProjectLintReport, Diagnostic } from 'newo-dsl-analyzer';
+import type { Reporter } from './types.js';
+
+export const sarifReporter: Reporter = {
+  write(report: ProjectLintReport): string {
+    const sarif = {
+      $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
+      version: '2.1.0',
+      runs: [
+        {
+          tool: {
+            driver: {
+              name: 'newo-lint',
+              informationUri: 'https://github.com/sabbah13/newo-cli',
+              rules: [] as Array<{ id: string }>,
+            },
+          },
+          results: report.results.flatMap(r =>
+            r.diagnostics.map(d => buildResult(r.filePath, d)),
+          ),
+        },
+      ],
+    };
+    return JSON.stringify(sarif, null, 2);
+  },
+};
+
+function buildResult(filePath: string, d: Diagnostic): Record<string, unknown> {
+  return {
+    ruleId: d.code,
+    level: d.severity === 'error' ? 'error' : d.severity === 'warning' ? 'warning' : 'note',
+    message: { text: d.message },
+    locations: [
+      {
+        physicalLocation: {
+          artifactLocation: { uri: toUri(filePath) },
+          region: {
+            startLine: d.range.start.line,
+            startColumn: d.range.start.column,
+            endLine: d.range.end.line,
+            endColumn: d.range.end.column,
+          },
+        },
+      },
+    ],
+  };
+}
+
+function toUri(absPath: string): string {
+  // SARIF artifact URIs should be workspace-relative when possible.
+  const rel = absPath.replace(process.cwd() + '/', '');
+  return rel.replace(/\\/g, '/');
+}

package/src/lint/reporters/text.ts

@@ -0,0 +1,58 @@
+/**
+ * Human-readable terminal reporter. Mirrors the ESLint 'stylish' layout:
+ *
+ *   path/to/file.jinja
+ *     12:5   error   Unknown skill: foo. Did you mean: bar?  E100
+ *     ...
+ *
+ *   2 problems (1 error, 1 warning)
+ */
+import path from 'path';
+import type { ProjectLintReport, LintResult } from 'newo-dsl-analyzer';
+import type { Reporter } from './types.js';
+
+const RED = '\x1b[31m';
+const YELLOW = '\x1b[33m';
+const CYAN = '\x1b[36m';
+const GREY = '\x1b[90m';
+const RESET = '\x1b[0m';
+const BOLD = '\x1b[1m';
+
+export const textReporter: Reporter = {
+  write(report: ProjectLintReport): string {
+    const lines: string[] = [];
+    const filesWithIssues = report.results.filter(r => r.diagnostics.length > 0);
+
+    for (const result of filesWithIssues) {
+      lines.push(renderFile(result));
+      lines.push('');
+    }
+
+    lines.push(renderSummary(report));
+    return lines.join('\n');
+  },
+};
+
+function renderFile(result: LintResult): string {
+  const rel = path.relative(process.cwd(), result.filePath);
+  const header = `${BOLD}${CYAN}${rel}${RESET}`;
+  const rows = result.diagnostics.map(d => {
+    const loc = `${d.range.start.line}:${d.range.start.column}`;
+    const sev = d.severity === 'error'
+      ? `${RED}error${RESET}`
+      : d.severity === 'warning'
+        ? `${YELLOW}warning${RESET}`
+        : `${GREY}${d.severity}${RESET}`;
+    return `  ${loc.padEnd(7)} ${sev.padEnd(16)} ${d.message}  ${GREY}${d.code}${RESET}`;
+  });
+  return [header, ...rows].join('\n');
+}
+
+function renderSummary(report: ProjectLintReport): string {
+  const total = report.errorCount + report.warningCount;
+  if (total === 0) {
+    return `${GREY}No issues found.${RESET}`;
+  }
+  const color = report.errorCount > 0 ? RED : YELLOW;
+  return `${color}${BOLD}${total} problems${RESET} (${report.errorCount} error${report.errorCount === 1 ? '' : 's'}, ${report.warningCount} warning${report.warningCount === 1 ? '' : 's'})`;
+}

package/src/lint/reporters/types.ts

@@ -0,0 +1,7 @@
+import type { ProjectLintReport } from 'newo-dsl-analyzer';
+
+export type ReporterName = 'text' | 'json' | 'sarif';
+
+export interface Reporter {
+  write(report: ProjectLintReport): string;
+}

package/src/sync/attributes.ts

@@ -12,6 +12,11 @@ import path from 'path';
 import fs from 'fs-extra';
 import yaml from 'js-yaml';
 import { patchYamlToPyyaml } from '../format/yaml-patch.js';
+import {
+  isJsonValueType,
+  normalizeJsonValueForStorage,
+  jsonValuesEqual
+} from './json-attr-utils.js';
 import type { AxiosInstance } from 'axios';
 import type { CustomerConfig } from '../types.js';
 
@@ -43,15 +48,21 @@ export async function saveCustomerAttributes(
         idMapping[attr.idn] = attr.id;
       }
 
-      // Special handling for complex JSON string values
+      // Coerce JSON-typed values to a STRING. The API can return the value
+      // as a parsed object for `value_type: json` attributes; if we let
+      // yaml.dump serialize that as a YAML structure the next push sends
+      // `{"value": {...}}` instead of `{"value": "..."}` and the Workflow
+      // Builder canvas breaks. See src/sync/json-attr-utils.ts for the
+      // full rationale.
       let processedValue = attr.value;
-      if (typeof attr.value === 'string' && attr.value.startsWith('[{') && attr.value.endsWith('}]')) {
+      if (isJsonValueType(attr.value_type)) {
+        processedValue = normalizeJsonValueForStorage(attr.value);
+      } else if (typeof attr.value === 'string' && attr.value.startsWith('[{') && attr.value.endsWith('}]')) {
+        // Legacy: reformat array-of-objects JSON strings for readability
         try {
-          // Parse and reformat JSON for better readability
           const parsed = JSON.parse(attr.value);
-          processedValue = JSON.stringify(parsed, null, 0); // No extra spacing, but valid JSON
+          processedValue = JSON.stringify(parsed, null, 0); // compact, valid JSON
         } catch (e) {
-          // Keep original if parsing fails
           processedValue = attr.value;
         }
       }
@@ -145,9 +156,12 @@ export async function saveProjectAttributes(
         idMapping[attr.idn] = attr.id;
       }
 
-      // Special handling for complex JSON string values
+      // Coerce JSON-typed values to a STRING. See json-attr-utils.ts for
+      // why this matters (Workflow Builder canvas blank-screen bug).
       let processedValue = attr.value;
-      if (typeof attr.value === 'string' && attr.value.startsWith('[{') && attr.value.endsWith('}]')) {
+      if (isJsonValueType(attr.value_type)) {
+        processedValue = normalizeJsonValueForStorage(attr.value);
+      } else if (typeof attr.value === 'string' && attr.value.startsWith('[{') && attr.value.endsWith('}]')) {
         try {
           const parsed = JSON.parse(attr.value);
           processedValue = JSON.stringify(parsed, null, 0);
@@ -297,19 +311,34 @@ export async function pushProjectAttributes(
 
     // Value type is already parsed (we removed !enum tags above)
     const valueType = localAttr.value_type;
-
-    // Check if value changed (use ?? to preserve 0, false, empty string)
-    const localValue = String(localAttr.value ?? '');
-    const remoteValue = String(remoteAttr.value ?? '');
-
-    if (localValue !== remoteValue) {
+    const isJson = isJsonValueType(valueType);
+
+    // Check if value changed.
+    // For JSON-typed values, compare canonical (compact) JSON so that
+    // pretty- vs compact-printed forms don't register as changes and
+    // string vs object representations compare equal. For everything
+    // else, fall back to the existing String() comparison (which still
+    // preserves 0, false, "" via ??).
+    const valuesAreEqual = isJson
+      ? jsonValuesEqual(localAttr.value, remoteAttr.value)
+      : String(localAttr.value ?? '') === String(remoteAttr.value ?? '');
+
+    if (!valuesAreEqual) {
       if (verbose) console.log(`   🔄 Updating project attribute: ${localAttr.idn}`);
 
       try {
+        // Always send JSON-typed values as a STRING. If the API or our
+        // YAML loader handed us an object, the platform stores it
+        // differently from the original string and the Workflow Builder
+        // canvas blanks out.
+        const valueToSend = isJson
+          ? normalizeJsonValueForStorage(localAttr.value)
+          : localAttr.value;
+
         const attributeToUpdate = {
           id: attributeId,
           idn: localAttr.idn,
-          value: localAttr.value,
+          value: valueToSend,
           title: localAttr.title,
           description: localAttr.description,
           group: localAttr.group,