newo 3.6.2 → 3.7.0

package/src/cli/commands/lint.ts

@@ -0,0 +1,246 @@
+/**
+ * `newo lint` - static analysis over Guidance / Jinja / NSL / NSLG files.
+ *
+ * Wraps newo-dsl-analyzer with newo-cli's customer/format/hash primitives.
+ * Exit codes:
+ *   0  clean (or only warnings below --max-warnings)
+ *   1  lint errors found, or warning threshold exceeded
+ *   2  unexpected runtime failure (handled by handleCliError)
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import {
+  createLinter,
+  type LinterOptions,
+  type ProjectLintReport,
+  type RuleSeverity,
+} from 'newo-dsl-analyzer';
+
+import { selectSingleCustomer } from '../customer-selection.js';
+import { handleCliError } from '../errors.js';
+import { resolveFormat } from '../../format/detect.js';
+import { discoverCustomerFiles, discoverFromPath, defaultRoot } from '../../lint/discovery.js';
+import { loadNewoLintConfig } from '../../lint/config.js';
+import { refreshLiveSchema, loadCachedLiveSchema } from '../../lint/live-schema.js';
+import { pickReporter } from '../../lint/reporters/index.js';
+import type { MultiCustomerConfig, CliArgs, CustomerConfig } from '../../types.js';
+
+interface LintArgs {
+  positional: string[];
+  formatVersion: string | undefined;
+  reporter: string;
+  maxWarnings: number;
+  quiet: boolean;
+  rules: string[];
+  noRules: string[];
+  changed: boolean;
+  live: boolean;
+  customer: string | undefined;
+}
+
+export async function handleLintCommand(
+  customerConfig: MultiCustomerConfig,
+  args: CliArgs,
+  verbose: boolean,
+): Promise<void> {
+  try {
+    const lintArgs = parseArgs(args);
+    const report = await run(customerConfig, lintArgs, verbose);
+    const reporter = pickReporter(lintArgs.reporter);
+    const output = reporter.write(report);
+    if (output.trim().length > 0) process.stdout.write(output + '\n');
+
+    const exitCode = determineExitCode(report, lintArgs);
+    if (exitCode !== 0) process.exit(exitCode);
+  } catch (err) {
+    handleCliError(err, 'lint');
+  }
+}
+
+function parseArgs(args: CliArgs): LintArgs {
+  const positional = args._.slice(1).filter((p): p is string => typeof p === 'string');
+
+  // Multiple flag shapes: --rule=E100, --rule E100,W100
+  // Disabling rules uses --rule-off (not --no-rule) because minimist treats
+  // `--no-X` as `X: false` and swallows the next positional argument.
+  const rules = collectCsv(args.rule);
+  const noRules = collectCsv(args['rule-off']);
+
+  return {
+    positional,
+    formatVersion: args.format as string | undefined,
+    reporter: (args.reporter as string | undefined) ?? (args['output-format'] as string | undefined) ?? 'text',
+    maxWarnings: parseIntOr(args['max-warnings'], Number.POSITIVE_INFINITY),
+    quiet: Boolean(args.quiet),
+    rules,
+    noRules,
+    changed: Boolean(args.changed),
+    live: Boolean(args.live),
+    customer: args.customer as string | undefined,
+  };
+}
+
+async function run(
+  customerConfig: MultiCustomerConfig,
+  args: LintArgs,
+  verbose: boolean,
+): Promise<ProjectLintReport> {
+  const config = loadNewoLintConfig();
+
+  // With explicit positional paths AND no customer/live flag, skip customer
+  // selection entirely - lint operates purely on the given filesystem paths.
+  const hasCustomerContext =
+    args.customer !== undefined ||
+    args.live ||
+    Object.keys(customerConfig.customers ?? {}).length > 0;
+
+  const { selectedCustomer, allCustomers, isMultiCustomer } = hasCustomerContext
+    ? selectSingleCustomer(customerConfig, args.customer)
+    : { selectedCustomer: null, allCustomers: [] as CustomerConfig[], isMultiCustomer: false };
+
+  const targetCustomer = selectedCustomer ?? (isMultiCustomer ? null : allCustomers[0] ?? null);
+
+  const schemas = await resolveSchemas(targetCustomer, args, verbose);
+
+  const ruleOverrides: Record<string, RuleSeverity> = {
+    ...(config.rules ?? {}),
+  };
+  for (const code of args.noRules) ruleOverrides[code] = 'off';
+  // --rule enables; we map unknown codes to 'warning' to avoid silently accepting typos.
+  for (const code of args.rules) {
+    if (!(code in ruleOverrides) || ruleOverrides[code] === 'off') {
+      ruleOverrides[code] = 'warning';
+    }
+  }
+
+  const linterOpts: LinterOptions = {
+    rules: ruleOverrides,
+    ...(schemas !== undefined ? { schemas } : {}),
+  };
+  const linter = createLinter(linterOpts);
+
+  const files = await resolveFiles(targetCustomer, allCustomers, args, isMultiCustomer);
+  if (files.length === 0) {
+    return { results: [], errorCount: 0, warningCount: 0 };
+  }
+
+  let errorCount = 0;
+  let warningCount = 0;
+  const results = [];
+  for (const file of files) {
+    const source = await fs.readFile(file.absPath, 'utf8');
+    const result = linter.lint(source, file.absPath);
+    for (const d of result.diagnostics) {
+      if (d.severity === 'error') errorCount++;
+      else if (d.severity === 'warning') warningCount++;
+    }
+    if (args.quiet) {
+      result.diagnostics = result.diagnostics.filter(d => d.severity === 'error');
+    }
+    results.push(result);
+  }
+  return { results, errorCount, warningCount: args.quiet ? 0 : warningCount };
+}
+
+async function resolveSchemas(
+  customer: CustomerConfig | null,
+  args: LintArgs,
+  verbose: boolean,
+): Promise<LinterOptions['schemas']> {
+  if (!customer) return 'bundled';
+
+  if (args.live) {
+    if (verbose) console.log(`Refreshing live schemas for ${customer.idn}...`);
+    const snapshot = await refreshLiveSchema(customer);
+    return { kind: 'inline', actions: snapshot.actions };
+  }
+
+  // Auto-use cached live snapshot if it exists (faster, always specific to
+  // the customer's actual NEWO account state). Fall back to bundled.
+  const cached = await loadCachedLiveSchema(customer.idn);
+  if (cached) {
+    if (verbose) {
+      const age = Math.round((Date.now() - Date.parse(cached.fetchedAt)) / 1000 / 60);
+      console.log(`Using cached schemas for ${customer.idn} (${age} min old). Use --live to refresh.`);
+    }
+    return { kind: 'inline', actions: cached.actions };
+  }
+
+  return 'bundled';
+}
+
+async function resolveFiles(
+  selected: CustomerConfig | null,
+  all: CustomerConfig[],
+  args: LintArgs,
+  isMultiCustomer: boolean,
+): ReturnType<typeof discoverFromPath> {
+  // Explicit positional paths beat everything else.
+  if (args.positional.length > 0) {
+    const files = [];
+    for (const p of args.positional) {
+      const discovered = await discoverFromPath(p, {
+        ...(args.formatVersion ? { format: toFormatVersion(args.formatVersion) } : {}),
+      });
+      files.push(...discovered);
+    }
+    return files;
+  }
+
+  if (selected) {
+    const formatVersion = resolveFormat(selected.idn, args.formatVersion).version;
+    return discoverCustomerFiles(selected, {
+      format: formatVersion,
+      changedOnly: args.changed,
+    });
+  }
+
+  if (isMultiCustomer) {
+    const files = [];
+    for (const customer of all) {
+      const formatVersion = resolveFormat(customer.idn, args.formatVersion).version;
+      const customerFiles = await discoverCustomerFiles(customer, {
+        format: formatVersion,
+        changedOnly: args.changed,
+      });
+      files.push(...customerFiles);
+    }
+    return files;
+  }
+
+  // No customer context - lint cwd / newo_customers/ directly.
+  return discoverFromPath(defaultRoot(), {
+    ...(args.formatVersion ? { format: toFormatVersion(args.formatVersion) } : {}),
+  });
+}
+
+function determineExitCode(report: ProjectLintReport, args: LintArgs): number {
+  if (report.errorCount > 0) return 1;
+  if (report.warningCount > args.maxWarnings) return 1;
+  return 0;
+}
+
+function collectCsv(value: unknown): string[] {
+  if (value === undefined || value === null) return [];
+  const items = Array.isArray(value) ? value : [value];
+  return items
+    .flatMap((v: unknown) => String(v).split(','))
+    .map(s => s.trim())
+    .filter(s => s.length > 0);
+}
+
+function parseIntOr(value: unknown, fallback: number): number {
+  if (typeof value === 'number' && Number.isFinite(value)) return value;
+  if (typeof value === 'string') {
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isNaN(parsed)) return parsed;
+  }
+  return fallback;
+}
+
+function toFormatVersion(v: string): 'cli_v1' | 'newo_v2' {
+  return v === 'newo_v2' ? 'newo_v2' : 'cli_v1';
+}
+
+// Silence unused path import warning; path is used via discovery helpers.
+void path;

package/src/cli.ts

@@ -47,19 +47,14 @@ import { handleWatchCommand } from './cli/commands/watch.js';
 import { handleDiffCommand } from './cli/commands/diff.js';
 import { handleLogsCommand } from './cli/commands/logs.js';
 import { handleExportCommand } from './cli/commands/export.js';
+import { handleLintCommand } from './cli/commands/lint.js';
+import { handleFormatCommand } from './cli/commands/format.js';
+import { handleCheckCommand } from './cli/commands/check.js';
 import type { CliArgs, NewoApiError } from './types.js';
 
 dotenv.config();
 
 async function main(): Promise<void> {
-  try {
-    // Initialize and validate environment at startup
-    initializeEnvironment();
-  } catch (error: unknown) {
-    console.error('Environment validation failed:', error instanceof Error ? error.message : String(error));
-    process.exit(1);
-  }
-
   const args = minimist(process.argv.slice(2)) as CliArgs;
   const cmd = args._[0];
   const verbose = Boolean(args.verbose || args.v);
@@ -72,12 +67,46 @@ async function main(): Promise<void> {
 
   if (verbose) console.log(`🔍 Command parsed: "${cmd}"`);
 
-  // Handle help command first
+  // Handle help command first - no env or customer config needed
   if (!cmd || ['help', '-h', '--help'].includes(cmd)) {
     handleHelpCommand();
     return;
   }
 
+  // Offline commands: lint/format/check don't need NEWO credentials
+  // UNLESS the user passes --customer or --live (both touch the API).
+  const isOfflineLint =
+    (cmd === 'lint' || cmd === 'format' || cmd === 'check') &&
+    !args.customer &&
+    !args.live;
+
+  if (isOfflineLint) {
+    try {
+      const emptyConfig: import('./types.js').MultiCustomerConfig = { customers: {} };
+      switch (cmd) {
+        case 'lint':
+          await handleLintCommand(emptyConfig, args, verbose);
+          return;
+        case 'format':
+          await handleFormatCommand(emptyConfig, args, verbose);
+          return;
+        case 'check':
+          await handleCheckCommand(emptyConfig, args, verbose);
+          return;
+      }
+    } catch (error: unknown) {
+      handleCliError(error, cmd);
+    }
+  }
+
+  // All other commands (and customer-scoped lint/format/check) require env + customer config.
+  try {
+    initializeEnvironment();
+  } catch (error: unknown) {
+    console.error('Environment validation failed:', error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  }
+
   // Handle list-customers command (doesn't need full customer config)
   if (cmd === 'list-customers') {
     try {
@@ -249,6 +278,18 @@ async function main(): Promise<void> {
         await handleDiffCommand(customerConfig, args, verbose);
         break;
 
+      case 'lint':
+        await handleLintCommand(customerConfig, args, verbose);
+        break;
+
+      case 'format':
+        await handleFormatCommand(customerConfig, args, verbose);
+        break;
+
+      case 'check':
+        await handleCheckCommand(customerConfig, args, verbose);
+        break;
+
       default:
         console.error('Unknown command:', cmd);
         console.error('Run "newo --help" for usage information');

package/src/lint/config.ts

@@ -0,0 +1,17 @@
+/**
+ * Lint config resolution for the newo CLI.
+ *
+ * Looks for `.neworc.yaml` / `.neworc.yml` / `.neworc.json` starting at
+ * the cwd and walking up to the filesystem root. Thin wrapper around
+ * newo-dsl-analyzer's `loadConfig` so consumers can override location
+ * per command if they need to.
+ */
+import path from 'path';
+import { loadConfig as analyzerLoadConfig } from 'newo-dsl-analyzer';
+import type { NewoLintConfig } from 'newo-dsl-analyzer';
+
+export type { NewoLintConfig } from 'newo-dsl-analyzer';
+
+export function loadNewoLintConfig(startDir: string = process.cwd()): NewoLintConfig {
+  return analyzerLoadConfig(path.resolve(startDir)) ?? {};
+}

package/src/lint/discovery.ts

@@ -0,0 +1,148 @@
+/**
+ * File discovery for `newo lint` / `newo format` / `newo check`.
+ *
+ * Walks a customer's tree (or any directory passed on the CLI), filters
+ * by format-aware extensions, and optionally narrows to files changed
+ * since the last push by consulting `.newo/{customer}/hashes.json`.
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import { NEWO_CUSTOMERS_DIR, customerDir } from '../fsutil.js';
+import { loadHashes } from '../hash.js';
+import { sha256 } from '../hash.js';
+import {
+  ALL_SCRIPT_EXTENSIONS,
+  type FormatVersion,
+  CLI_V1_EXTENSIONS,
+  NEWO_V2_EXTENSIONS,
+} from '../format/types.js';
+import type { CustomerConfig } from '../types.js';
+
+export interface DiscoveryOptions {
+  /** If set, restrict to one format's extensions only. */
+  format?: FormatVersion;
+  /** If true, include only files that hash-differ from `.newo/{customer}/hashes.json`. */
+  changedOnly?: boolean;
+  /** Additional absolute paths to skip. */
+  ignore?: string[];
+}
+
+export interface DiscoveredFile {
+  /** Absolute path on disk. */
+  absPath: string;
+  /** Path relative to the customer root (for display / reports). */
+  relPath: string;
+  /** Extension (including leading dot). */
+  ext: string;
+}
+
+/**
+ * Discover script files under a customer's tree.
+ * Respects format when given (else walks all recognized extensions).
+ */
+export async function discoverCustomerFiles(
+  customer: CustomerConfig,
+  opts: DiscoveryOptions = {},
+): Promise<DiscoveredFile[]> {
+  const root = customerDir(customer.idn);
+  if (!(await fs.pathExists(root))) return [];
+
+  const exts = pickExtensions(opts.format);
+  const ignoreSet = new Set(opts.ignore ?? []);
+
+  const hits = await walkForExtensions(root, exts, ignoreSet);
+
+  if (!opts.changedOnly) {
+    return hits.map(absPath => toDiscoveredFile(absPath, root));
+  }
+
+  const stored = await loadHashes(customer.idn);
+  const changed: DiscoveredFile[] = [];
+  for (const absPath of hits) {
+    const current = sha256(await fs.readFile(absPath, 'utf8'));
+    if (stored[absPath] !== current) {
+      changed.push(toDiscoveredFile(absPath, root));
+    }
+  }
+  return changed;
+}
+
+/**
+ * Discover files under an arbitrary directory.
+ * Used when the user passes explicit paths to `newo lint some/dir`.
+ */
+export async function discoverFromPath(
+  inputPath: string,
+  opts: DiscoveryOptions = {},
+): Promise<DiscoveredFile[]> {
+  const abs = path.resolve(inputPath);
+  if (!(await fs.pathExists(abs))) return [];
+
+  const exts = pickExtensions(opts.format);
+  const ignoreSet = new Set(opts.ignore ?? []);
+  const stat = await fs.stat(abs);
+
+  let hits: string[];
+  if (stat.isFile()) {
+    hits = exts.includes(path.extname(abs)) ? [abs] : [];
+  } else {
+    hits = await walkForExtensions(abs, exts, ignoreSet);
+  }
+
+  const root = stat.isDirectory() ? abs : path.dirname(abs);
+  return hits.map(p => toDiscoveredFile(p, root));
+}
+
+function pickExtensions(format?: FormatVersion): string[] {
+  if (!format) return [...ALL_SCRIPT_EXTENSIONS];
+  const map = format === 'newo_v2' ? NEWO_V2_EXTENSIONS : CLI_V1_EXTENSIONS;
+  return Object.values(map);
+}
+
+function toDiscoveredFile(absPath: string, root: string): DiscoveredFile {
+  return {
+    absPath,
+    relPath: path.relative(root, absPath),
+    ext: path.extname(absPath),
+  };
+}
+
+async function walkForExtensions(
+  dir: string,
+  exts: string[],
+  ignore: Set<string>,
+): Promise<string[]> {
+  const out: string[] = [];
+  const stack: string[] = [dir];
+  while (stack.length > 0) {
+    const current = stack.pop()!;
+    if (ignore.has(current)) continue;
+    let entries: fs.Dirent[];
+    try {
+      entries = await fs.readdir(current, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      // Skip hidden dirs, node_modules, and the .newo state directory.
+      if (entry.name.startsWith('.')) continue;
+      if (entry.name === 'node_modules') continue;
+      const full = path.join(current, entry.name);
+      if (ignore.has(full)) continue;
+      if (entry.isDirectory()) {
+        stack.push(full);
+      } else if (exts.includes(path.extname(entry.name))) {
+        out.push(full);
+      }
+    }
+  }
+  return out.sort();
+}
+
+/**
+ * Default root for lint invocations with no path arguments - the
+ * `newo_customers/` directory at the cwd.
+ */
+export function defaultRoot(): string {
+  return NEWO_CUSTOMERS_DIR;
+}

package/src/lint/live-schema.ts

@@ -0,0 +1,62 @@
+/**
+ * Live schema refresh: hits `/api/v1/script/actions` via the existing
+ * NEWO api client, caches the response to `.newo/{customer}/actions.json`,
+ * and returns an object shaped for `createLinter({ schemas: { kind: 'inline', ... }})`.
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import { customerStateDir } from '../fsutil.js';
+import type { CustomerConfig, ScriptAction } from '../types.js';
+import { getValidAccessToken } from '../auth.js';
+import { makeClient, getScriptActions } from '../api.js';
+
+export interface LiveSchemaSnapshot {
+  actions: Array<{ name: string; [k: string]: unknown }>;
+  fetchedAt: string;
+}
+
+export async function liveSchemaCachePath(customerIdn: string): Promise<string> {
+  const dir = customerStateDir(customerIdn);
+  await fs.ensureDir(dir);
+  return path.join(dir, 'actions.json');
+}
+
+/**
+ * Fetch the current action catalog from NEWO and cache it.
+ * Returns a snapshot ready to pass to `createLinter`.
+ */
+export async function refreshLiveSchema(
+  customer: CustomerConfig,
+): Promise<LiveSchemaSnapshot> {
+  const token = await getValidAccessToken(customer);
+  const client = await makeClient(false, token);
+  const actions = await getScriptActions(client);
+  const snapshot: LiveSchemaSnapshot = {
+    actions: actions.map((a: ScriptAction) => ({
+      name: a.idn ?? a.title,
+      title: a.title,
+      ...(a.idn !== undefined ? { idn: a.idn } : {}),
+      arguments: a.arguments,
+    })),
+    fetchedAt: new Date().toISOString(),
+  };
+  const cachePath = await liveSchemaCachePath(customer.idn);
+  await fs.writeJson(cachePath, snapshot, { spaces: 2 });
+  return snapshot;
+}
+
+/**
+ * Load the cached snapshot if present. Returns null when the cache is
+ * missing or corrupt (caller should fall back to bundled schemas).
+ */
+export async function loadCachedLiveSchema(
+  customerIdn: string,
+): Promise<LiveSchemaSnapshot | null> {
+  const cachePath = await liveSchemaCachePath(customerIdn);
+  if (!(await fs.pathExists(cachePath))) return null;
+  try {
+    return (await fs.readJson(cachePath)) as LiveSchemaSnapshot;
+  } catch {
+    return null;
+  }
+}

package/src/lint/reporters/index.ts

@@ -0,0 +1,22 @@
+import { textReporter } from './text.js';
+import { jsonReporter } from './json.js';
+import { sarifReporter } from './sarif.js';
+import type { Reporter, ReporterName } from './types.js';
+
+export type { Reporter, ReporterName } from './types.js';
+
+export function pickReporter(name: ReporterName | string | undefined): Reporter {
+  switch (name) {
+    case 'json':
+      return jsonReporter;
+    case 'sarif':
+      return sarifReporter;
+    case 'text':
+    case undefined:
+    case '':
+      return textReporter;
+    default:
+      console.warn(`Unknown --format value '${name}', defaulting to text.`);
+      return textReporter;
+  }
+}

package/src/lint/reporters/json.ts

@@ -0,0 +1,12 @@
+/**
+ * Machine-readable JSON reporter. Matches the ProjectLintReport shape
+ * exactly - consumers can parse with any JSON tool.
+ */
+import type { ProjectLintReport } from 'newo-dsl-analyzer';
+import type { Reporter } from './types.js';
+
+export const jsonReporter: Reporter = {
+  write(report: ProjectLintReport): string {
+    return JSON.stringify(report, null, 2);
+  },
+};

package/src/lint/reporters/sarif.ts

@@ -0,0 +1,59 @@
+/**
+ * SARIF 2.1.0 reporter - lets GitHub Advanced Security / Code Scanning
+ * pick up newo-lint findings alongside CodeQL and other linters.
+ *
+ * Spec: https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html
+ */
+import type { ProjectLintReport, Diagnostic } from 'newo-dsl-analyzer';
+import type { Reporter } from './types.js';
+
+export const sarifReporter: Reporter = {
+  write(report: ProjectLintReport): string {
+    const sarif = {
+      $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
+      version: '2.1.0',
+      runs: [
+        {
+          tool: {
+            driver: {
+              name: 'newo-lint',
+              informationUri: 'https://github.com/sabbah13/newo-cli',
+              rules: [] as Array<{ id: string }>,
+            },
+          },
+          results: report.results.flatMap(r =>
+            r.diagnostics.map(d => buildResult(r.filePath, d)),
+          ),
+        },
+      ],
+    };
+    return JSON.stringify(sarif, null, 2);
+  },
+};
+
+function buildResult(filePath: string, d: Diagnostic): Record<string, unknown> {
+  return {
+    ruleId: d.code,
+    level: d.severity === 'error' ? 'error' : d.severity === 'warning' ? 'warning' : 'note',
+    message: { text: d.message },
+    locations: [
+      {
+        physicalLocation: {
+          artifactLocation: { uri: toUri(filePath) },
+          region: {
+            startLine: d.range.start.line,
+            startColumn: d.range.start.column,
+            endLine: d.range.end.line,
+            endColumn: d.range.end.column,
+          },
+        },
+      },
+    ],
+  };
+}
+
+function toUri(absPath: string): string {
+  // SARIF artifact URIs should be workspace-relative when possible.
+  const rel = absPath.replace(process.cwd() + '/', '');
+  return rel.replace(/\\/g, '/');
+}