@bevel-software/platform-core-backend 0.13.6 → 0.15.1

package/src/shared/__tests__/kb-walk.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import { walkKb, type KbWalkListener } from '../kb-walk.js';
+
+/**
+ * The one walk: what it skips, in what order it visits, what it calls a hole
+ * — and that every listener on it is told exactly the same things.
+ */
+describe('walkKb', () => {
+  let root: string;
+  const write = async (rel: string, text = '') => {
+    const abs = path.join(root, rel);
+    await fs.mkdir(path.dirname(abs), { recursive: true });
+    await fs.writeFile(abs, text);
+  };
+  /** A listener that records every event it is told, in order. */
+  const recorder = () => {
+    const events: string[] = [];
+    const listener: KbWalkListener = {
+      onDir: (rel, entries) => void events.push(`dir ${rel || '.'} [${entries.map((e) => e.name).join(' ')}]`),
+      onFile: (dir, name) => void events.push(`file ${dir ? `${dir}/${name}` : name}`),
+      onHole: (rel) => void events.push(`hole ${rel || '.'}`),
+    };
+    return { events, listener };
+  };
+
+  beforeEach(async () => {
+    root = await fs.mkdtemp(path.join(os.tmpdir(), 'kb-walk-'));
+  });
+  afterEach(() => fs.rm(root, { recursive: true, force: true }));
+
+  it('visits every folder once, in component order, skipping dot-entries and node_modules', async () => {
+    await write('Plugins/a-b/plugin.json');
+    await write('Plugins/a/b/plugin.json');
+    await write('KnowledgeBase/x.md');
+    await write('.git/HEAD');
+    await write('node_modules/dep/index.js');
+    await write('Plugins/.parked/plugin.json');
+    const { events, listener } = recorder();
+    const { holes } = await walkKb(root, [listener]);
+    expect(holes).toEqual([]);
+    expect(events).toEqual([
+      'dir . [KnowledgeBase Plugins]',
+      'dir KnowledgeBase [x.md]',
+      'file KnowledgeBase/x.md',
+      'dir Plugins [a a-b]', // "a" before "a-b": the walk order every consumer agrees on
+      'dir Plugins/a [b]',
+      'dir Plugins/a/b [plugin.json]',
+      'file Plugins/a/b/plugin.json',
+      'dir Plugins/a-b [plugin.json]',
+      'file Plugins/a-b/plugin.json',
+    ]);
+  });
+
+  it('tells every listener the same things', async () => {
+    await write('Plugins/GTM/plugin.json');
+    await write('Skills/Eng/deploy/SKILL.md');
+    const a = recorder();
+    const b = recorder();
+    await walkKb(root, [a.listener, b.listener]);
+    expect(a.events.length).toBeGreaterThan(0);
+    expect(b.events).toEqual(a.events);
+  });
+
+  it('a folder that exists but cannot be listed is a hole — reported to every listener and in the result', async () => {
+    await write('Plugins/locked/plugin.json');
+    await write('Plugins/open/plugin.json');
+    const real = fs.readdir;
+    const spy = vi.spyOn(fs, 'readdir').mockImplementation(((dir: string, opts: unknown) =>
+      String(dir).endsWith('locked')
+        ? Promise.reject(Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' }))
+        : (real as (d: string, o: unknown) => Promise<unknown>).call(fs, dir, opts)) as never);
+    try {
+      const { events, listener } = recorder();
+      const { holes } = await walkKb(root, [listener]);
+      expect(holes).toEqual(['Plugins/locked']);
+      expect(events).toContain('hole Plugins/locked');
+      // The rest of the tree is still walked.
+      expect(events).toContain('file Plugins/open/plugin.json');
+    } finally {
+      spy.mockRestore();
+    }
+  });
+
+  it('a missing root is an empty walk; a folder that vanished between listing and visiting is not a hole', async () => {
+    const empty = recorder();
+    expect(await walkKb(path.join(root, 'nope'), [empty.listener])).toEqual({ holes: [] });
+    expect(empty.events).toEqual([]);
+
+    await write('Plugins/gone/plugin.json');
+    const real = fs.readdir;
+    const spy = vi.spyOn(fs, 'readdir').mockImplementation(((dir: string, opts: unknown) =>
+      String(dir).endsWith('gone')
+        ? Promise.reject(Object.assign(new Error('ENOENT: no such file'), { code: 'ENOENT' }))
+        : (real as (d: string, o: unknown) => Promise<unknown>).call(fs, dir, opts)) as never);
+    try {
+      const { events, listener } = recorder();
+      const { holes } = await walkKb(root, [listener]);
+      expect(holes).toEqual([]);
+      expect(events.some((e) => e.startsWith('hole'))).toBe(false);
+    } finally {
+      spy.mockRestore();
+    }
+  });
+});

package/src/shared/__tests__/path-order.test.ts

@@ -0,0 +1,24 @@
+import { describe, it, expect } from 'vitest';
+import { comparePathComponents } from '../path-order.js';
+
+describe('comparePathComponents', () => {
+  it('orders component by component, the way a sorted directory walk visits paths', () => {
+    // A whole-string compare would weigh "-" against "/" and put a-b/x first.
+    expect(comparePathComponents('a/b/x', 'a-b/x')).toBeLessThan(0);
+    expect(comparePathComponents('Plugins/GTM', 'Plugins/GTM/skills')).toBeLessThan(0);
+    expect(comparePathComponents('Plugins/GTM', 'Plugins/GTM')).toBe(0);
+  });
+
+  it('never calls two distinct paths equal: a collation tie falls through to code-unit order', () => {
+    // Precomposed é and its decomposed spelling collate as equal; two folders
+    // so named must still have ONE first, or discovery and principal synthesis
+    // could each claim a different one for the same slug.
+    const composed = 'Plugins/café';
+    const decomposed = 'Plugins/café';
+    // (Whether the host's collation equates them is the host's business —
+    // the property under test holds either way: distinct paths never tie.)
+    const order = comparePathComponents(composed, decomposed);
+    expect(order).not.toBe(0);
+    expect(comparePathComponents(decomposed, composed)).toBe(-order);
+  });
+});

package/src/shared/__tests__/token-crypto.test.ts

@@ -0,0 +1,34 @@
+import { describe, it, expect } from 'vitest';
+import { TokenCrypto } from '../token-crypto.js';
+import { randomBytes } from 'node:crypto';
+
+/**
+ * The contract an operator meets at boot: a wrong-length key must name the
+ * variable THEY set. The error used to hardcode `SHAREPOINT_TOKEN_ENC_KEY` —
+ * the primitive's birthplace — and a core deployment failing on its vault key
+ * was told to fix a SharePoint variable it never configured.
+ */
+describe('TokenCrypto key validation', () => {
+  // 30 bytes — the classic wrong key: 40 base64 chars instead of 44.
+  const SHORT = randomBytes(30).toString('base64');
+
+  it('names SECRETS_ENC_KEY by default — the variable every core consumer reads', () => {
+    expect(() => new TokenCrypto(SHORT)).toThrowError(
+      /^SECRETS_ENC_KEY must decode to 32 bytes \(got 30\)/,
+    );
+  });
+
+  it('names the variable an overlay passes for its own key', () => {
+    expect(() => new TokenCrypto(SHORT, 'SHAREPOINT_TOKEN_ENC_KEY')).toThrowError(
+      /^SHAREPOINT_TOKEN_ENC_KEY must decode to 32 bytes/,
+    );
+  });
+
+  it('accepts a 32-byte key as base64 and as hex, and round-trips', () => {
+    const raw = randomBytes(32);
+    for (const encoded of [raw.toString('base64'), raw.toString('hex')]) {
+      const crypto = new TokenCrypto(encoded);
+      expect(crypto.decrypt(crypto.encrypt('s3cret'))).toBe('s3cret');
+    }
+  });
+});

package/src/shared/__tests__/ttl-cache.test.ts

@@ -0,0 +1,55 @@
+import { describe, it, expect } from 'vitest';
+import { TtlCache } from '../ttl-cache.js';
+
+describe('TtlCache', () => {
+  it('serves a value until the TTL runs out, then stops', () => {
+    let now = 0;
+    const cache = new TtlCache<string>(1000, () => now);
+    cache.set('v', cache.begin());
+    expect(cache.get()).toBe('v');
+    now = 999;
+    expect(cache.get()).toBe('v');
+    now = 1000;
+    expect(cache.get()).toBeNull();
+  });
+
+  it('invalidate() drops the value immediately', () => {
+    const cache = new TtlCache<string>(1000, () => 0);
+    cache.set('v', cache.begin());
+    cache.invalidate();
+    expect(cache.get()).toBeNull();
+  });
+
+  /**
+   * The race the token exists for, and the one that made the merge fix a
+   * no-op in exactly the case it targeted: a catalog read starts on the
+   * pre-merge tree, the merge's pull lands and drops the cache, and the
+   * in-flight read then writes its stale list back with a fresh full TTL —
+   * silently undoing the drop.
+   */
+  it('discards a value whose fetch straddled an invalidate()', () => {
+    const cache = new TtlCache<string>(1000, () => 0);
+    const token = cache.begin(); // read starts on the old tree
+    cache.invalidate(); //           the pull lands mid-read
+    cache.set('stale', token); //    the read comes back and tries to store
+    expect(cache.get()).toBeNull();
+  });
+
+  it('stores a value whose fetch did not straddle an invalidate()', () => {
+    const cache = new TtlCache<string>(1000, () => 0);
+    cache.invalidate();
+    const token = cache.begin();
+    cache.set('fresh', token);
+    expect(cache.get()).toBe('fresh');
+  });
+
+  it('keeps refusing a token from before the drop, not just the first time', () => {
+    const cache = new TtlCache<string>(1000, () => 0);
+    const stale = cache.begin();
+    cache.invalidate();
+    const fresh = cache.begin();
+    cache.set('fresh', fresh);
+    cache.set('stale', stale);
+    expect(cache.get()).toBe('fresh');
+  });
+});

package/src/shared/domain-errors.ts

@@ -166,6 +166,35 @@ export class PullRebaseConflictError extends WorkflowDomainError {
   }
 }
 
+/**
+ * The clone's branch no longer exists on origin: the fetch that refreshes
+ * `refs/remotes/origin/<branch>` found no such ref. Distinct from an
+ * unreachable remote — the host answered, and the answer was "gone" — so a
+ * caller can treat the clone as stale rather than the sync as failed. 410.
+ */
+export class RemoteBranchGoneError extends WorkflowDomainError {
+  readonly kind = 'remote-branch-gone' as const;
+  constructor(readonly branch: string) {
+    super(`Branch "${branch}" no longer exists on the remote.`, 410, {
+      kind: 'remote-branch-gone',
+      branch,
+    });
+    this.name = 'RemoteBranchGoneError';
+  }
+}
+
+/**
+ * Whether a failed clone or fetch failed because origin has no such branch, as
+ * opposed to being unreachable or refusing the credential. Git's wording for
+ * the two shapes: `Remote branch <x> not found in upstream origin` (clone) and
+ * `couldn't find remote ref refs/heads/<x>` (fetch). The ONE classifier for
+ * this fact — the workspace bootstrap and the git layer both consult it, so
+ * a wording learned by one is learned by both.
+ */
+export function isMissingRemoteBranchFailure(message: string): boolean {
+  return /Remote branch .* not found in upstream|couldn't find remote ref/i.test(message);
+}
+
 /**
  * Generic 400 for workflow-input validation (malformed branch names, missing
  * fields, etc.). Carries an optional payload so callers can attach typed

package/src/shared/fs-errors.ts

@@ -0,0 +1,10 @@
+/**
+ * Whether a filesystem error means "there is no such path" — the one failure
+ * a reader may treat as an ordinary answer. Everything else (permissions,
+ * I/O, a loop) is a failure to READ what is there, and a caller that folds
+ * it into absence turns an outage into a wrong answer.
+ */
+export function isAbsence(err: unknown): boolean {
+  const code = (err as { code?: unknown } | null)?.code;
+  return code === 'ENOENT' || code === 'ENOTDIR';
+}

package/src/shared/fs-walk.ts

@@ -1,30 +1,45 @@
-import path from 'node:path';
-import fs from 'node:fs/promises';
+import { walkKb, isSkippedEntry } from './kb-walk.js';
+
+export { isSkippedEntry };
 
 /**
- * The ONE recursive file walk for catalog scanners: relative (`/`-separated,
- * sorted) paths of files under `root` whose basename matches. A missing root
- * yields `[]`; dot-entries (incl. `.git`) are skipped. Scanners with different
- * semantics (the skills scanner's stop-at-skill-folder walk, the access
- * resolver's dir chains) deliberately keep their own walks.
+ * The file listing catalog scanners want, on top of {@link walkKb}: relative
+ * (`/`-separated, sorted) paths of files under `root` whose basename matches.
+ * A missing root yields `[]`; skipped entries are never entered.
+ *
+ * A directory that cannot be listed is SKIPPED by default — right for a
+ * catalog, which shows what it can. A caller that must see EVERYTHING or
+ * nothing passes `strict`, and the listing throws a {@link WalkError} naming
+ * the directory instead: a list with a hole in it is not a list of every
+ * file, and the caller can say WHICH hole rather than pass a raw errno up.
  */
-export async function walkFiles(root: string, match: (basename: string) => boolean): Promise<string[]> {
+export async function walkFiles(
+  root: string,
+  match: (basename: string) => boolean,
+  opts: { strict?: boolean } = {},
+): Promise<string[]> {
   const out: string[] = [];
-  const walk = async (dir: string, rel: string): Promise<void> => {
-    let entries: import('node:fs').Dirent[];
-    try {
-      entries = await fs.readdir(dir, { withFileTypes: true });
-    } catch {
-      return;
-    }
-    for (const entry of entries) {
-      if (entry.name.startsWith('.')) continue;
-      const childRel = rel ? `${rel}/${entry.name}` : entry.name;
-      if (entry.isDirectory()) await walk(path.join(dir, entry.name), childRel);
-      else if (entry.isFile() && match(entry.name)) out.push(childRel);
-    }
-  };
-  await walk(root, '');
+  await walkKb(root, [
+    {
+      onFile(dir, name) {
+        if (match(name)) out.push(dir ? `${dir}/${name}` : name);
+      },
+      onHole(rel, err) {
+        if (opts.strict) throw new WalkError(rel, err);
+      },
+    },
+  ]);
   out.sort();
   return out;
 }
+
+/** A strict listing's refusal: the directory (relative to the root, `''` for the root itself) it could not list. */
+export class WalkError extends Error {
+  constructor(
+    readonly relDir: string,
+    readonly cause: unknown,
+  ) {
+    super(`${relDir || '.'} could not be listed — ${cause instanceof Error ? cause.message : String(cause)}`);
+    this.name = 'WalkError';
+  }
+}

package/src/shared/kb-walk.ts

@@ -0,0 +1,95 @@
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import { isAbsence } from './fs-errors.js';
+import { comparePathComponents } from './path-order.js';
+
+/**
+ * THE walk of a knowledge-base checkout.
+ *
+ * Every reader of the tree — the access resolver looking for rules, plugin
+ * discovery looking for manifests, a rename looking for grants, the catalog
+ * scanners — used to carry a recursive `readdir` loop of its own, each with
+ * its own skip list, its own idea of what an unreadable folder means and its
+ * own visiting order. Two loops that disagree on one of those are two
+ * answers to "what is in the tree", and every such disagreement was a bug.
+ *
+ * So there is one loop, and it decides three things once:
+ *
+ *   - WHAT IS SKIPPED: {@link isSkippedEntry} — dot-entries (`.git`, a parked
+ *     delete) and vendored dependencies. Nothing else is pruned; a listener
+ *     that stops at a leaf of its own (a skill folder, a plugin folder)
+ *     simply ignores what lies beneath it.
+ *   - IN WHAT ORDER: each folder's entries in {@link comparePathComponents}
+ *     order, so "first by path" means the same thing to every listener.
+ *   - WHAT A HOLE IS: a folder that exists but could not be listed is
+ *     reported as an event and in the result — never thrown, never silently
+ *     skipped — so a reader can show what it saw and a writer can refuse.
+ *     A folder that vanished between listing and visiting is not a hole.
+ *
+ * Listeners see the same events in the same order; what each makes of them
+ * is its own business (`.bevelignore` included — the explorer honours it,
+ * the resolver must not, so it is a listener's decision, not the walk's).
+ */
+export function isSkippedEntry(name: string): boolean {
+  return name.startsWith('.') || name === 'node_modules';
+}
+
+/** A directory entry as the walk presents it: a name and what it is. */
+export interface WalkedEntry {
+  name: string;
+  isDirectory(): boolean;
+  isFile(): boolean;
+}
+
+export interface KbWalkListener {
+  /** A folder was listed. `rel` is repo-relative (`''` for the root); `entries` are its children, skipped ones removed, in walk order. */
+  onDir?(rel: string, entries: readonly WalkedEntry[]): void | Promise<void>;
+  /** A file was seen. `dir` is its folder (`''` for the root), `name` its basename. */
+  onFile?(dir: string, name: string): void | Promise<void>;
+  /** A folder that exists but could not be listed. */
+  onHole?(rel: string, err: unknown): void | Promise<void>;
+}
+
+export interface KbWalkResult {
+  /** Every folder that could not be listed, repo-relative (`''` for the root). */
+  holes: string[];
+}
+
+/**
+ * Walk the checkout at `root`, driving every listener from one traversal.
+ * A missing root is an empty tree. Listener errors propagate untouched.
+ */
+export async function walkKb(root: string, listeners: readonly KbWalkListener[]): Promise<KbWalkResult> {
+  const holes: string[] = [];
+
+  const visit = async (abs: string, rel: string): Promise<void> => {
+    let listed: import('node:fs').Dirent[];
+    try {
+      listed = await fs.readdir(abs, { withFileTypes: true });
+    } catch (err) {
+      if (isAbsence(err)) {
+        // The root: a checkout without this tree, an empty walk. Deeper: a
+        // folder that vanished between listing and visiting — not a hole.
+        return;
+      }
+      holes.push(rel);
+      for (const l of listeners) await l.onHole?.(rel, err);
+      return;
+    }
+    const entries: WalkedEntry[] = listed
+      .filter((e) => !isSkippedEntry(e.name) && (e.isDirectory() || e.isFile()))
+      .sort((a, b) => comparePathComponents(a.name, b.name));
+    for (const l of listeners) await l.onDir?.(rel, entries);
+    for (const entry of entries) {
+      const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        await visit(path.join(abs, entry.name), childRel);
+      } else {
+        for (const l of listeners) await l.onFile?.(rel, entry.name);
+      }
+    }
+  };
+
+  await visit(root, '');
+  return { holes };
+}

package/src/shared/path-order.ts

@@ -0,0 +1,33 @@
+/**
+ * The ONE ordering of repository paths: component by component, each
+ * component by `localeCompare`, a shorter path before its extensions. It is
+ * exactly the order a directory walk that sorts each level's entries visits
+ * them in — so a consumer that must agree with the walk on "first by path"
+ * (plugin discovery keeps the first of two folders sharing a slug; principal
+ * synthesis must pick the same one) sorts with this and nothing else. A
+ * whole-string `localeCompare` does not agree: it weighs the separator
+ * against the next character (`a-b/x` before `a/b/x`), the walk never does.
+ *
+ * TOTAL over distinct paths: two components the collation cannot tell apart
+ * (a precomposed `é` and its decomposed spelling) fall through to code-unit
+ * order, so no pair of distinct paths ever compares equal — an equal pair
+ * would leave "first" to whichever enumeration happened to run, and the two
+ * consumers could then pick different folders for one slug.
+ */
+export function comparePathComponents(a: string, b: string): number {
+  const as = a.split('/');
+  const bs = b.split('/');
+  const n = Math.min(as.length, bs.length);
+  for (let i = 0; i < n; i++) {
+    const order = compareComponent(as[i]!, bs[i]!);
+    if (order !== 0) return order;
+  }
+  return as.length - bs.length;
+}
+
+/** One component against another: collation first, code units to break a tie. */
+function compareComponent(a: string, b: string): number {
+  const order = a.localeCompare(b);
+  if (order !== 0) return order;
+  return a < b ? -1 : a > b ? 1 : 0;
+}

package/src/shared/token-crypto.ts

@@ -11,16 +11,16 @@ import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto';
 export class TokenCrypto {
   private readonly key: Buffer;
 
-  /** @param rawKey 32-byte key as hex (64 chars) or base64. */
-  constructor(rawKey: string) {
-    const key = decodeKey(rawKey);
-    if (key.length !== 32) {
-      throw new Error(
-        `SHAREPOINT_TOKEN_ENC_KEY must decode to 32 bytes (got ${key.length}). ` +
-          'Generate one with: `node -e "console.log(require(\'crypto\').randomBytes(32).toString(\'base64\'))"`.',
-      );
-    }
-    this.key = key;
+  /**
+   * @param rawKey 32-byte key as hex (64 chars) or base64.
+   * @param envVarName the environment variable the caller read `rawKey` from,
+   *   so a bad key is reported against the variable the operator actually has
+   *   to fix. Defaults to core's own `SECRETS_ENC_KEY` — every core consumer
+   *   reads that; an overlay bringing its own key (the SharePoint token
+   *   cache, connector configs) passes its own name.
+   */
+  constructor(rawKey: string, envVarName = 'SECRETS_ENC_KEY') {
+    this.key = assertKeyDecodesTo32Bytes(rawKey, envVarName);
   }
 
   encrypt(plaintext: string): string {
@@ -52,3 +52,21 @@ function decodeKey(raw: string): Buffer {
   if (/^[0-9a-fA-F]{64}$/.test(raw)) return Buffer.from(raw, 'hex');
   return Buffer.from(raw, 'base64');
 }
+
+/**
+ * Decode a key and refuse anything that is not exactly 32 bytes, blaming
+ * `envVarName`. Exported so boot-time config can validate the key THE MOMENT
+ * it knows which environment variable supplied it (`SECRETS_ENC_KEY` or a
+ * legacy fallback) — every later `new TokenCrypto(key)` with the default name
+ * is then safe, because a bad key never gets that far.
+ */
+export function assertKeyDecodesTo32Bytes(rawKey: string, envVarName: string): Buffer {
+  const key = decodeKey(rawKey);
+  if (key.length !== 32) {
+    throw new Error(
+      `${envVarName} must decode to 32 bytes (got ${key.length}). ` +
+        'Generate one with: `node -e "console.log(require(\'crypto\').randomBytes(32).toString(\'base64\'))"`.',
+    );
+  }
+  return key;
+}

package/src/shared/ttl-cache.ts

@@ -6,6 +6,11 @@
  */
 export class TtlCache<T> {
   private entry: { at: number; value: T } | null = null;
+  /**
+   * Bumped by every `invalidate()`. A fetch carries the value this had when it
+   * started, and `set` refuses a value whose fetch straddled a drop — see there.
+   */
+  private generation = 0;
 
   constructor(
     private readonly ttlMs: number,
@@ -17,11 +22,38 @@ export class TtlCache<T> {
     return null;
   }
 
-  set(value: T): void {
+  /**
+   * Take a token BEFORE starting the fetch whose result you will `set`.
+   *
+   * Every caller of this cache reads the disk across an `await`, and the whole
+   * point of `invalidate()` is that it fires during exactly that window: the
+   * merge pulls the new tree in while a catalog read that started on the old
+   * one is still in flight. Without a token that late `set` wins, and the cache
+   * holds the PRE-invalidation scan for a fresh full TTL — the drop silently
+   * undone, and the staleness it existed to prevent restored.
+   */
+  begin(): number {
+    return this.generation;
+  }
+
+  /**
+   * Store a value fetched under `token`. Required, not optional: a caller that
+   * could forget it is a caller that reintroduces the race above, and there are
+   * few enough call sites to make the token cheap.
+   *
+   * A stale token means an `invalidate()` landed mid-fetch, so this value
+   * describes a tree that is already gone: it is DISCARDED rather than stored.
+   * The fetcher still returns it to its own caller — that read is merely as old
+   * as the moment it started, which is true of any read — but the next caller
+   * re-scans instead of inheriting it.
+   */
+  set(value: T, token: number): void {
+    if (token !== this.generation) return;
     this.entry = { at: this.now(), value };
   }
 
   invalidate(): void {
     this.entry = null;
+    this.generation++;
   }
 }