flecto 2.1.0 → 3.0.1

package/package.json

@@ -4,8 +4,8 @@
     "access": "public",
     "provenance": true
   },
-  "version": "2.1.0",
-  "description": "Flecto — semantic config watcher that reports meaningful changes in plain English",
+  "version": "3.0.1",
+  "description": "Flecto \u2014 semantic config watcher that reports meaningful changes in plain English",
   "license": "MIT",
   "keywords": [
     "flecto",
@@ -43,6 +43,7 @@
   "scripts": {
     "test": "node --test test/*.test.js",
     "test:watch": "node --test --watch test/*.test.js",
+    "bench": "node bench/run.js",
     "pack:check": "npm pack --dry-run"
   },
   "dependencies": {

package/schemas/flecto-policy-pack-2.0.json

@@ -7,6 +7,7 @@
   "required": ["rules"],
   "properties": {
     "id": { "type": "string", "minLength": 1 },
+    "expandSubtrees": { "type": "boolean" },
     "rules": {
       "type": "array",
       "items": { "$ref": "#/$defs/rule" }
@@ -41,6 +42,8 @@
         "afterIn": { "type": "array" },
         "beforeTruthy": { "const": true },
         "afterTruthy": { "const": true },
+        "beforeLooksSecret": { "const": true },
+        "afterLooksSecret": { "const": true },
         "afterMatches": { "type": "string" },
         "numericJump": {
           "type": "object",
@@ -89,6 +92,8 @@
         "afterIn": { "type": "array" },
         "beforeTruthy": { "const": true },
         "afterTruthy": { "const": true },
+        "beforeLooksSecret": { "const": true },
+        "afterLooksSecret": { "const": true },
         "afterMatches": { "type": "string" },
         "numericJump": { "$ref": "#/$defs/numericJump" },
         "numericDelta": { "$ref": "#/$defs/numericDelta" }

package/src/alerter.js

@@ -2,6 +2,7 @@ import { spawn } from 'child_process';
 import { mkdirSync, writeFileSync, readFileSync, readdirSync, unlinkSync } from 'fs';
 import { resolve } from 'path';
 import { renderWarn } from './renderer.js';
+import { formatWebhookPayload } from './notifiers.js';
 
 const ALERT_TMP_DIR = '.flecto-tmp';
 const ALERT_QUEUE_DIR = '.flecto-queue';
@@ -135,11 +136,19 @@ async function flushPersistentQueue(deliver) {
 /**
  * @param {string} url
  * @param {import('./envelope.js').FlectoEnvelope} envelope
- * @param {{ headers?: Record<string, string>, timeoutMs?: number, retries?: number }} [options]
+ * @param {{
+ *  headers?: Record<string, string>,
+ *  timeoutMs?: number,
+ *  retries?: number,
+ *  format?: import('./notifiers.js').WebhookFormat
+ * }} [options]
  * @returns {Promise<boolean>}
  */
 export async function postWebhook(url, envelope, options = {}) {
-  const body = JSON.stringify(envelope);
+  // `flecto` (the default) returns the envelope untouched, so the body is
+  // byte-identical to what has always been posted. Chat formats reshape only
+  // the body: headers, retries, and delivery modes are unchanged.
+  const body = JSON.stringify(formatWebhookPayload(envelope, options.format ?? 'flecto'));
   const timeoutMs = options.timeoutMs ?? 5_000;
   const retries = options.retries ?? 2;
   const headers = {
@@ -186,7 +195,13 @@ export async function postWebhook(url, envelope, options = {}) {
 }
 
 /**
- * @param {{ webhook?: string, webhookHeaders?: Record<string, string>, webhookTimeoutMs?: number, webhookRetries?: number }} options
+ * @param {{
+ *  webhook?: string,
+ *  webhookHeaders?: Record<string, string>,
+ *  webhookTimeoutMs?: number,
+ *  webhookRetries?: number,
+ *  webhookFormat?: import('./notifiers.js').WebhookFormat
+ * }} options
  * @param {import('./envelope.js').FlectoEnvelope} envelope
  */
 async function deliverWebhook(options, envelope) {
@@ -195,6 +210,7 @@ async function deliverWebhook(options, envelope) {
     headers: options.webhookHeaders,
     timeoutMs: options.webhookTimeoutMs,
     retries: options.webhookRetries,
+    format: options.webhookFormat,
   });
 }
 
@@ -217,6 +233,7 @@ function applyFailurePolicy(options, ok) {
  *  webhookHeaders?: Record<string, string>,
  *  webhookTimeoutMs?: number,
  *  webhookRetries?: number,
+ *  webhookFormat?: import('./notifiers.js').WebhookFormat,
  *  deliveryMode?: 'best-effort' | 'at-least-once',
  *  onAlertFailure?: 'warn' | 'exit' | 'retry'
  * }} options

package/src/config.js

@@ -1,9 +1,14 @@
-import { existsSync, readFileSync, writeFileSync } from 'fs';
-import { resolve } from 'path';
+import { existsSync, readFileSync, readdirSync, writeFileSync } from 'fs';
+import { resolve, sep } from 'path';
 import fg from 'fast-glob';
 import yaml from 'js-yaml';
+import { isEnvFilename } from './parser.js';
 
 const RC_CANDIDATES = ['.flectorc', '.flectorc.json', '.flectorc.yaml', '.flectorc.yml'];
+const COMPOSE_FILENAMES = ['docker-compose.yml', 'docker-compose.yaml', 'compose.yml', 'compose.yaml'];
+const CONFIG_DIR_PATTERN = 'config/**/*.{yaml,yml,json,toml,ini}';
+const ENV_FILE_PATTERNS = ['.env', '.env.*', '*.env'];
+const GENERIC_FILE_PATTERNS = [CONFIG_DIR_PATTERN, ...ENV_FILE_PATTERNS];
 
 /**
  * @typedef {{
@@ -13,6 +18,15 @@ const RC_CANDIDATES = ['.flectorc', '.flectorc.json', '.flectorc.yaml', '.flecto
  *  include?: string[],
  *  exclude?: string[]
  * }} FlectoRc
+ *
+ * @typedef {{
+ *  id: string,
+ *  evidence: string[],
+ *  pack: string | null,
+ *  summary: string
+ * }} StackSignal
+ *
+ * @typedef {{ signals: StackSignal[], packs: string[], files: string[] }} StackDetection
  */
 
 /**
@@ -66,24 +80,70 @@ export function resolveEffectiveOptions(config, profile, cliOverrides = {}) {
   return { ...defaults, ...profileOptions, ...cliOverrides };
 }
 
+/**
+ * Has the operator explicitly opted in to plugins declared in `.flectorc`?
+ * @returns {boolean}
+ */
+function rcPluginsAllowed() {
+  const raw = process.env.FLECTO_ALLOW_RC_PLUGINS;
+  return raw === '1' || String(raw).toLowerCase() === 'true';
+}
+
+/**
+ * Split a policy list that may arrive as an array or a comma-separated string.
+ * @param {unknown} raw
+ * @param {string[]} fallback
+ * @returns {string[]}
+ */
+function toList(raw, fallback) {
+  if (Array.isArray(raw)) return raw.map(String);
+  if (typeof raw === 'string') return raw.split(',').map((s) => s.trim()).filter(Boolean);
+  return fallback;
+}
+
 /**
  * Normalize policy-related effective options.
+ *
+ * Plugins execute code, and `.flectorc` is attacker-controlled on an untrusted
+ * pull request, so a plugin that came from the rc file rather than an explicit
+ * `--plugins` flag is refused unless the operator opts in with
+ * `FLECTO_ALLOW_RC_PLUGINS=1`. Refusing loudly rather than skipping silently is
+ * deliberate: a plugin that stops running without saying so would weaken a
+ * policy gate the operator believes is in place.
  * @param {Record<string, unknown>} effective
+ * @param {{ pluginsFromCli?: boolean, cwd?: string }} [provenance]
  */
-export function resolvePolicyOptions(effective) {
-  const policiesRaw = effective.policies;
-  const pluginsRaw = effective.plugins;
+export function resolvePolicyOptions(effective, provenance = {}) {
   const severityRemapRaw = effective.severityRemap;
-  const policies = Array.isArray(policiesRaw)
-    ? policiesRaw.map(String)
-    : typeof policiesRaw === 'string'
-      ? String(policiesRaw).split(',').map((s) => s.trim()).filter(Boolean)
-      : ['default'];
-  const plugins = Array.isArray(pluginsRaw)
-    ? pluginsRaw.map(String)
-    : typeof pluginsRaw === 'string'
-      ? String(pluginsRaw).split(',').map((s) => s.trim()).filter(Boolean)
-      : [];
+  const policies = toList(effective.policies, ['default']);
+  const plugins = toList(effective.plugins, []);
+
+  if (plugins.length > 0 && !provenance.pluginsFromCli) {
+    if (!rcPluginsAllowed()) {
+      throw new Error(
+        'Refusing to load policy plugins declared in .flectorc: plugins execute code, '
+        + 'and a config file can come from an untrusted pull request.\n'
+        + `Declared: ${plugins.join(', ')}\n`
+        + 'Pass them on the command line with --plugins instead, or set '
+        + 'FLECTO_ALLOW_RC_PLUGINS=1 if this config is trusted.',
+      );
+    }
+    // Opted in, but the rc file may still be attacker-authored. Keep rc-declared
+    // plugins inside the project so `../../../../tmp/x.mjs` cannot reach a module
+    // planted elsewhere on the runner. An explicit --plugins is operator intent
+    // and stays unrestricted: shared policies outside the cwd are a real setup.
+    const root = resolve(provenance.cwd ?? process.cwd());
+    for (const pluginPath of plugins) {
+      const abs = resolve(root, pluginPath);
+      if (abs !== root && !abs.startsWith(root + sep)) {
+        throw new Error(
+          `Policy plugin declared in .flectorc is outside the project: ${pluginPath}\n`
+          + 'Plugins execute code, so an rc-declared plugin must live inside the '
+          + 'directory Flecto is running in.',
+        );
+      }
+    }
+  }
   if (
     severityRemapRaw !== undefined
     && (severityRemapRaw === null || Array.isArray(severityRemapRaw) || typeof severityRemapRaw !== 'object')
@@ -126,13 +186,104 @@ export async function resolveFiles(input) {
 }
 
 /**
- * Scaffold a starter rc file if missing.
+ * Detect stack signals in a directory and map them to policy packs and file
+ * patterns. Only built-in pack ids and patterns Flecto can actually parse are
+ * ever returned, so a config built from this stays loadable by every command.
+ * Terraform files are reported as context only: `.tf` is not a supported parse
+ * format, and the `terraform` pack reads plan JSON rather than `.tf` sources,
+ * so neither belongs in a config built from a directory listing.
  * @param {string} cwd
- * @returns {string}
+ * @returns {StackDetection}
+ */
+export function detectStack(cwd = process.cwd()) {
+  /** @type {StackSignal[]} */
+  const signals = [];
+  const packs = ['default'];
+  /** @type {string[]} */
+  const files = [];
+
+  let entries = [];
+  try {
+    entries = readdirSync(cwd, { withFileTypes: true });
+  } catch {
+    return { signals, packs, files };
+  }
+  const fileNames = entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
+  const dirNames = new Set(entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name));
+  const fileNameSet = new Set(fileNames);
+
+  const composeFiles = COMPOSE_FILENAMES.filter((name) => fileNameSet.has(name));
+  if (composeFiles.length > 0) {
+    packs.push('compose');
+    files.push(...composeFiles);
+    signals.push({
+      id: 'compose',
+      evidence: composeFiles,
+      pack: 'compose',
+      summary: `Detected ${composeFiles.join(', ')} → enabled the \`compose\` policy pack and watched ${composeFiles.length === 1 ? 'it' : 'them'}`,
+    });
+  }
+
+  if (fileNameSet.has('package.json')) {
+    packs.push('node-runtime');
+    files.push('package.json');
+    signals.push({
+      id: 'node',
+      evidence: ['package.json'],
+      pack: 'node-runtime',
+      summary: 'Detected package.json → enabled the `node-runtime` policy pack and watched it',
+    });
+  }
+
+  const terraformFiles = fileNames.filter((name) => name.toLowerCase().endsWith('.tf')).sort();
+  if (terraformFiles.length > 0) {
+    signals.push({
+      id: 'terraform',
+      evidence: terraformFiles,
+      pack: null,
+      summary: `Detected Terraform files (${terraformFiles.join(', ')}) → .tf is not a parseable format, so nothing was enabled; run "flecto plan" on "terraform show -json" output to use the terraform pack`,
+    });
+  }
+
+  if (dirNames.has('config')) {
+    files.push(CONFIG_DIR_PATTERN);
+    signals.push({
+      id: 'config-dir',
+      evidence: ['config/'],
+      pack: null,
+      summary: `Detected config/ → watched ${CONFIG_DIR_PATTERN}`,
+    });
+  }
+
+  const envFiles = fileNames.filter((name) => isEnvFilename(name)).sort();
+  if (envFiles.length > 0) {
+    files.push(...ENV_FILE_PATTERNS);
+    signals.push({
+      id: 'dotenv',
+      evidence: envFiles,
+      pack: null,
+      summary: `Detected ${envFiles.join(', ')} → watched ${ENV_FILE_PATTERNS.join(', ')}`,
+    });
+  }
+
+  return { signals, packs, files: [...new Set(files)] };
+}
+
+/**
+ * Scaffold a starter rc file if missing, pre-selecting policy packs and file
+ * patterns from the stack signals found in `cwd`. Never overwrites an existing
+ * config: any of the four `.flectorc` candidates is reported back untouched.
+ * @param {string} cwd
+ * @returns {{ path: string, created: boolean, detection: StackDetection }}
  */
 export function initRcFile(cwd = process.cwd()) {
+  const detection = detectStack(cwd);
+  const existingPath = RC_CANDIDATES
+    .map((candidate) => resolve(cwd, candidate))
+    .find((candidate) => existsSync(candidate));
+  if (existingPath) return { path: existingPath, created: false, detection };
+
   const path = resolve(cwd, '.flectorc.json');
-  if (existsSync(path)) return path;
   const starter = {
     defaults: {
       mode: 'compact',
@@ -140,7 +291,7 @@ export function initRcFile(cwd = process.cwd()) {
       ignore: ['**.updated_at'],
       deliveryMode: 'best-effort',
       onAlertFailure: 'warn',
-      policies: ['default'],
+      policies: detection.packs,
       plugins: [],
       arrayIdKey: null,
       arrayId: true,
@@ -151,14 +302,14 @@ export function initRcFile(cwd = process.cwd()) {
       dev: { mode: 'verbose' },
       ci: { failOn: 'policy,error' },
       prod: {
-        policies: ['default', 'strict-prod'],
+        policies: [...detection.packs, 'strict-prod'],
         severityRemap: { 'pool-size-jump': 'error' },
         maskSecrets: true,
       },
     },
-    files: ['config/**/*.{yaml,yml,json,toml,ini}', '.env', '.env.*', '*.env'],
+    files: detection.files.length > 0 ? detection.files : [...GENERIC_FILE_PATTERNS],
     exclude: ['**/node_modules/**'],
   };
   writeFileSync(path, JSON.stringify(starter, null, 2), 'utf8');
-  return path;
+  return { path, created: true, detection };
 }

package/src/differ.js

@@ -1,14 +1,61 @@
+import { annotateEncryptedChanges } from './encrypted.js';
+import { diffDocumentKeys, stripDocumentPrefix } from './documents.js';
+
 /**
  * @typedef {{ type: 'added' | 'removed' | 'changed', path: string, before?: unknown, after?: unknown, note?: string }} ChangeEvent
  */
 
+/**
+ * Where an event's path carries a synthetic multi-document prefix, the same
+ * path without it. Recorded non-enumerably so the event stays byte-identical
+ * everywhere it is serialized — JSON output, webhook payloads, snapshots.
+ */
+const SECRET_MATCH_PATH = Symbol.for('flecto.secretMatchPath');
+
+/**
+ * The path to match secret-looking *key names* against.
+ *
+ * For an ordinary file this is just `event.path`. For a multi-document file it
+ * is the path with the document identity removed, because that identity is a
+ * resource name — user data — and a Deployment called `token-service` must not
+ * make every value inside it read as a secret. See documents.js.
+ * @param {ChangeEvent} event
+ * @returns {string}
+ */
+export function secretMatchPath(event) {
+  const stripped = /** @type {Record<string | symbol, unknown>} */ (event)?.[SECRET_MATCH_PATH];
+  return typeof stripped === 'string' ? stripped : (event?.path ?? '');
+}
+
+/**
+ * @param {ChangeEvent[]} events
+ * @param {readonly string[]} documentKeys
+ * @returns {ChangeEvent[]}
+ */
+function tagSecretMatchPaths(events, documentKeys) {
+  if (documentKeys.length === 0) return events;
+  for (const event of events) {
+    const stripped = stripDocumentPrefix(event.path, documentKeys);
+    if (stripped === event.path) continue;
+    Object.defineProperty(event, SECRET_MATCH_PATH, {
+      value: stripped,
+      enumerable: false,
+      writable: false,
+      configurable: true,
+    });
+  }
+  return events;
+}
+
 /**
  * Checks whether a value is a plain object (not array, not null).
  * @param {unknown} v
  * @returns {v is Record<string, unknown>}
  */
 function isPlainObject(v) {
-  return v !== null && typeof v === 'object' && !Array.isArray(v);
+  if (v === null || typeof v !== 'object' || Array.isArray(v)) return false;
+  const prototype = Object.getPrototypeOf(v);
+  return prototype === Object.prototype || prototype === null;
 }
 
 /**
@@ -410,5 +457,15 @@ export function diffTrees(before, after, options = {}) {
     diffValues(before, after, '<root>', events, diffOpts);
   }
 
-  return events.filter(e => !ignore(e.path));
+  // Encrypted files carry signals a key-by-key walk cannot express: the file
+  // gaining or losing encryption, and a MAC that moved on its own. This is a
+  // no-op — the same array, untouched — when neither side is encrypted.
+  // Ignore patterns are applied afterwards so `--ignore` silences the derived
+  // paths exactly like any other.
+  const annotated = annotateEncryptedChanges(before, after, events).filter(e => !ignore(e.path));
+
+  // Multi-document paths carry a resource name in front. Record what secret-name
+  // matching should look at, once, here — the renderers see events and nothing
+  // else, and a resource name must never decide whether a value gets masked.
+  return tagSecretMatchPaths(annotated, diffDocumentKeys(before, after));
 }

package/src/documents.js

@@ -0,0 +1,106 @@
+/**
+ * Carrying the "this tree is a multi-document wrapper" signal.
+ *
+ * A `---`-separated YAML file parses to a synthetic object keyed by document
+ * identity (see `parseYamlStream`), so every path inside a document gains a
+ * `Kind/namespace/name` prefix. Several subsystems need to know which leading
+ * path segments are those synthetic keys rather than real configuration keys —
+ * most importantly secret-name matching, which must never look at a resource
+ * name, because a resource *named* `token-service` is not a secret.
+ *
+ * The parser is the only thing that can answer this without guessing: it
+ * invented the keys. Rather than re-deriving them downstream from the shape of
+ * the keys — a heuristic that misfires the first time somebody writes a config
+ * whose top-level keys genuinely look like `Kind/ns/name` — the parser records
+ * them here and everything else reads them back.
+ *
+ * The record is a non-enumerable symbol property, so it is invisible to
+ * `Object.keys`, `JSON.stringify`, spread, and `assert.deepEqual`: a marked tree
+ * is byte-for-byte the same snapshot, diff, and payload it was before. The flip
+ * side is that it does not survive a JSON round trip, so a tree read back out of
+ * a snapshot file carries the keys only if the snapshot recorded them.
+ * `documentKeysOf` distinguishes the two: `[]` means "known not to be a
+ * wrapper", `null` means "provenance unknown".
+ */
+
+/** Where the synthetic document keys are recorded on a parsed tree. */
+const DOCUMENT_KEYS = Symbol.for('flecto.documentKeys');
+
+/** @type {readonly string[]} */
+const NONE = Object.freeze([]);
+
+/**
+ * Record the synthetic document keys the parser invented for a tree.
+ *
+ * Returns the tree itself — marking is a side effect on an object Flecto just
+ * created, never a copy. Non-object roots (a scalar YAML document, an opaque
+ * age blob) cannot carry the mark and are returned unchanged.
+ * @template T
+ * @param {T} tree
+ * @param {readonly string[] | null | undefined} keys
+ * @returns {T}
+ */
+export function withDocumentKeys(tree, keys) {
+  if (tree === null || typeof tree !== 'object') return tree;
+  Object.defineProperty(tree, DOCUMENT_KEYS, {
+    value: keys == null ? null : Object.freeze([...keys]),
+    enumerable: false,
+    writable: false,
+    configurable: true,
+  });
+  return tree;
+}
+
+/**
+ * The synthetic document keys recorded on a tree.
+ * @param {unknown} tree
+ * @returns {readonly string[] | null} `[]` when the tree is known to be a single
+ *   document, `null` when nothing recorded its provenance
+ */
+export function documentKeysOf(tree) {
+  if (tree === null || typeof tree !== 'object') return null;
+  const keys = /** @type {Record<string | symbol, unknown>} */ (tree)[DOCUMENT_KEYS];
+  return Array.isArray(keys) ? /** @type {readonly string[]} */ (keys) : null;
+}
+
+/**
+ * The document keys covering either side of a diff, longest first.
+ *
+ * A document present on only one side still prefixes the paths of its own
+ * additions or removals, so the union is what a path can start with. Longest
+ * first so that if one identity is a prefix of another — `app` and `app.web` —
+ * a path is attributed to the more specific one.
+ * @param {unknown} before
+ * @param {unknown} after
+ * @returns {readonly string[]}
+ */
+export function diffDocumentKeys(before, after) {
+  const merged = [...(documentKeysOf(before) ?? NONE), ...(documentKeysOf(after) ?? NONE)];
+  if (merged.length === 0) return NONE;
+  return [...new Set(merged)].sort((a, b) => b.length - a.length);
+}
+
+/**
+ * Drop a leading document-identity segment from a diff path.
+ *
+ * Matching is by whole segment against the keys the parser actually invented,
+ * not by pattern, so `Deployment/prod/token-service.spec.replicas` becomes
+ * `spec.replicas` while an ordinary key that merely begins with the same text
+ * is left alone. Returns the path unchanged when no key applies, which is the
+ * single-document case and therefore the common one.
+ * @param {string} path
+ * @param {readonly string[] | null | undefined} keys tried in order; see
+ *   {@link diffDocumentKeys} for why callers pass them longest first
+ * @returns {string}
+ */
+export function stripDocumentPrefix(path, keys) {
+  if (!keys || keys.length === 0 || !path) return path;
+  for (const key of keys) {
+    if (!key || !path.startsWith(key)) continue;
+    if (path.length === key.length) return '';
+    const next = path[key.length];
+    if (next === '.') return path.slice(key.length + 1);
+    if (next === '[') return path.slice(key.length);
+  }
+  return path;
+}