@bevel-software/platform-core-backend 0.14.0 → 0.19.0

package/src/modules/kb-sync/kb-sync.service.ts

@@ -0,0 +1,204 @@
+import type { BranchSyncOutcome } from '@bevel-software/platform-shared';
+import { logger } from '../../shared/logging.js';
+
+const log = logger('sync');
+import { sanitizeError } from '../workflow/sanitize-error.js';
+import type {
+  IKbSyncService,
+  LastSync,
+  SyncRequest,
+  SyncResult,
+  SyncWorkflowPort,
+  SyncWorkspacePort,
+} from './kb-sync.interface.js';
+
+/**
+ * Runs remote syncs one at a time and folds everything that arrives
+ * mid-run into ONE follow-up.
+ *
+ * Why coalesce: a git host fires several hooks for one event — Azure DevOps
+ * sends a push, a PR-updated and a PR-merged within a second of each other —
+ * and each would otherwise queue behind the workspace mutex to pull the same
+ * tree again. The follow-up pulls the union of every request that arrived
+ * during the run (an "all" absorbs the rest).
+ *
+ * But each caller is answered for the branches IT asked about. The endpoint
+ * promises to fail exactly when the caller's branch is not in sync; a hook
+ * for branch A answered with branch B's conflict would fail the wrong
+ * subscription — and a host that disables subscriptions after repeated
+ * failures would disable the wrong one.
+ *
+ * Branches are pulled sequentially. N is small (only cloned branches count),
+ * the per-clone mutex serialises them anyway, and one slow origin round-trip
+ * at a time keeps the process calm during a burst.
+ */
+export class KbSyncService implements IKbSyncService {
+  private inFlight: Promise<unknown> | null = null;
+  /** Everyone who arrived mid-run, each with what they asked for. */
+  private waiters: Array<{
+    branches: string[] | 'all';
+    by: string | undefined;
+    resolve: (r: SyncResult) => void;
+    reject: (e: unknown) => void;
+  }> = [];
+  private last: LastSync | null = null;
+
+  constructor(
+    private readonly workflow: SyncWorkflowPort,
+    private readonly workspaces: SyncWorkspacePort,
+    private readonly now: () => number = () => Date.now(),
+  ) {}
+
+  lastSync(): LastSync | null {
+    return this.last;
+  }
+
+  sync(request: SyncRequest): Promise<SyncResult> {
+    if (!this.inFlight) return this.start(request);
+    return new Promise<SyncResult>((resolve, reject) => {
+      // Copied on entry: the caller keeps its array, and a caller that reuses
+      // or edits it before the follow-up runs must not change what is pulled
+      // or what it is answered for.
+      const branches = request.branches === 'all' ? 'all' : [...request.branches];
+      this.waiters.push({ branches, by: request.by, resolve, reject });
+    });
+  }
+
+  private start(request: SyncRequest): Promise<SyncResult> {
+    const run = this.run(request);
+    this.inFlight = run.catch(() => undefined).finally(() => {
+      this.inFlight = null;
+      this.drain();
+    });
+    return run;
+  }
+
+  /**
+   * Run ONE follow-up for everyone who arrived during the last run: the union
+   * of their branches, then each waiter answered from its own share of the
+   * result — never from another caller's branches.
+   */
+  private drain(): void {
+    if (this.waiters.length === 0) return;
+    const waiters = this.waiters;
+    this.waiters = [];
+
+    const union = new Set<string>();
+    let all = false;
+    const by = new Set<string>();
+    for (const w of waiters) {
+      if (w.branches === 'all') all = true;
+      else for (const b of w.branches) union.add(b);
+      if (w.by) by.add(w.by);
+    }
+    const request: SyncRequest = {
+      branches: all ? 'all' : [...union],
+      by: by.size > 0 ? [...by].join(', ') : undefined,
+    };
+    this.start(request).then(
+      (result) => {
+        for (const w of waiters) w.resolve(KbSyncService.subsetFor(result, w.branches));
+      },
+      (err) => {
+        for (const w of waiters) w.reject(err);
+      },
+    );
+  }
+
+  /** The part of a union run that answers one caller: its branches, and its status over them. */
+  private static subsetFor(result: SyncResult, branches: string[] | 'all'): SyncResult {
+    if (branches === 'all') return result;
+    const wanted = new Set(branches);
+    const results = result.results.filter((r) => wanted.has(r.branch));
+    return {
+      status: KbSyncService.statusOf(results),
+      results,
+      changeRequests: result.changeRequests,
+    };
+  }
+
+  private static statusOf(results: BranchSyncOutcome[]): SyncResult['status'] {
+    const clean = results.every(
+      (r) =>
+        r.outcome === 'updated' ||
+        r.outcome === 'up-to-date' ||
+        r.outcome === 'not-cloned' ||
+        r.outcome === 'remote-gone',
+    );
+    return clean ? 'synced' : 'partial';
+  }
+
+  private async run(request: SyncRequest): Promise<SyncResult> {
+    const known = await this.workspaces.listClonedWorkspaces();
+    const byBranch = new Map(known.map((w) => [w.branch, w.id]));
+
+    const targets: Array<{ branch: string; id: string | null }> =
+      request.branches === 'all'
+        ? known.map((w) => ({ branch: w.branch, id: w.id }))
+        : request.branches.map((branch) => ({ branch, id: byBranch.get(branch) ?? null }));
+
+    const unreadable = new Map(known.filter((w) => w.unreadable).map((w) => [w.branch, w.unreadable!]));
+
+    const results: BranchSyncOutcome[] = [];
+    for (const { branch, id } of targets) {
+      if (id === null) {
+        results.push({ branch, outcome: 'not-cloned' });
+        continue;
+      }
+      const cannotRead = unreadable.get(branch);
+      if (cannotRead) {
+        results.push({ branch, outcome: 'error', error: cannotRead });
+        continue;
+      }
+      let outcome: BranchSyncOutcome;
+      try {
+        outcome = await this.workflow.syncWorkspaceFromRemote(id);
+      } catch (err) {
+        // The workflow step promises outcomes, not throws — but one broken
+        // clone must never take the other branches, the sweep, or every
+        // coalesced caller down with it, so the promise is enforced here too.
+        const message = sanitizeError(err);
+        log.error(`unexpected failure syncing "${branch}": ${message}`);
+        outcome = { branch, outcome: 'error', error: message };
+      }
+      results.push(outcome);
+      if (outcome.outcome === 'remote-gone') {
+        // The host deleted the branch. Left in place, the stale clone would
+        // fail its fetch on every following full sync — and a webhook host
+        // disables a subscription that keeps failing. The workflow retires it
+        // under the branch-lifecycle lock, after asking origin again, so a
+        // branch recreated since the pull keeps its clone. Best-effort.
+        try {
+          const removed = await this.workflow.retireRemoteGoneClone(id);
+          if (removed) log.info(`removed the clone of "${branch}" — deleted on the host`);
+          else log.info(`kept the clone of "${branch}" — it is back on the host, or already gone`);
+        } catch (err) {
+          log.warn(`could not remove the clone of "${branch}":`, { detail: sanitizeError(err) });
+        }
+      }
+    }
+
+    // The host may have deleted a branch (an ADO PR completed with "delete
+    // source branch"); the existing sweep closes the Hexis request that
+    // pointed at it. Best-effort — it fails safe by closing nothing.
+    let closedDeletedBranch = 0;
+    try {
+      closedDeletedBranch = await this.workflow.closeChangeRequestsWithDeletedBranches();
+    } catch (err) {
+      log.warn('deleted-branch sweep failed:', { err });
+    }
+
+    const result: SyncResult = {
+      status: KbSyncService.statusOf(results),
+      results,
+      changeRequests: { closedDeletedBranch },
+    };
+    this.last = {
+      at: this.now(),
+      by: request.by ?? 'unknown',
+      status: result.status,
+      results: [...results],
+    };
+    return result;
+  }
+}

package/src/modules/kb-sync/sync-auth.ts

@@ -0,0 +1,118 @@
+import { createHmac } from 'node:crypto';
+import { timingSafeStringEqual } from '../auth/password-hash.js';
+
+/**
+ * Who may call `POST /api/sync`, decided from the request's credentials.
+ *
+ * Framework-agnostic on purpose (the route hands in plain strings and the
+ * raw body) so the rule is one function with one test, not something spread
+ * across middleware.
+ *
+ * Four credentials are accepted, because the callers cannot all speak the
+ * same one:
+ *
+ *   bearer            `Authorization: Bearer <secret>` — ADO Web Hooks and
+ *                     any pipeline can set a header.
+ *   gitlab-token      `X-Gitlab-Token: <secret>` — what GitLab sends.
+ *   github-signature  `X-Hub-Signature-256: sha256=<hmac of the raw body>` —
+ *                     GitHub and Gitea cannot set headers, only sign.
+ *   admin-session     a browser JWT whose user is an admin, so a "Sync now"
+ *                     button and a curl with your own session both work while
+ *                     the hook is being wired up.
+ *
+ * Every comparison is constant-time. The secret never travels in the URL.
+ */
+export type SyncCredential =
+  | { kind: 'bearer' | 'gitlab-token' | 'github-signature' }
+  | { kind: 'admin-session'; email: string };
+
+export type SyncAuthResult =
+  | { ok: true; credential: SyncCredential }
+  | { ok: false; status: 401 | 503; message: string };
+
+export interface SyncAuthInput {
+  /** The configured secret, or empty when none is set. */
+  secret: string;
+  authorization?: string;
+  gitlabToken?: string;
+  hubSignature?: string;
+  /** The request body exactly as received — the HMAC is over these bytes. */
+  rawBody?: Buffer;
+}
+
+export interface SyncSessionVerifier {
+  /** Resolve a browser JWT to its email, or null when it is not one. */
+  verifyJwt(token: string): { email: string } | null;
+  isAdmin(email: string): Promise<boolean>;
+}
+
+const NO_SECRET =
+  'No sync secret is configured. Set KB_SYNC_SECRET (or the setup screen’s sync secret), ' +
+  'or call this endpoint with an administrator session.';
+
+/**
+ * Shortest secret the endpoint will honour. The setup screen enforces the
+ * same floor on save; this is the boundary that also covers a secret set
+ * through the environment, which no validator ever sees.
+ */
+export const MIN_SYNC_SECRET_LENGTH = 16;
+
+const SECRET_TOO_SHORT =
+  `The configured sync secret is shorter than ${MIN_SYNC_SECRET_LENGTH} characters and is not ` +
+  'accepted. Set a longer KB_SYNC_SECRET, or call this endpoint with an administrator session.';
+
+function signatureMatches(secret: string, rawBody: Buffer, header: string): boolean {
+  const [scheme, hex] = header.split('=', 2);
+  if (scheme !== 'sha256' || !hex) return false;
+  const expected = createHmac('sha256', secret).update(rawBody).digest('hex');
+  return timingSafeStringEqual(expected, hex.toLowerCase());
+}
+
+export async function verifySyncCredential(
+  input: SyncAuthInput,
+  session: SyncSessionVerifier,
+): Promise<SyncAuthResult> {
+  const configured = input.secret.trim();
+  // A too-short secret is treated as absent for matching purposes and named
+  // as the problem below, so a one-character deployment secret can never be
+  // what lets a caller in.
+  const usable = configured.length >= MIN_SYNC_SECRET_LENGTH;
+  const secret = usable ? configured : '';
+  const bearer = input.authorization?.startsWith('Bearer ')
+    ? input.authorization.slice('Bearer '.length).trim()
+    : '';
+
+  // A shared secret presented in any of its three shapes.
+  if (secret) {
+    if (bearer && timingSafeStringEqual(bearer, secret)) {
+      return { ok: true, credential: { kind: 'bearer' } };
+    }
+    if (input.gitlabToken && timingSafeStringEqual(input.gitlabToken.trim(), secret)) {
+      return { ok: true, credential: { kind: 'gitlab-token' } };
+    }
+    if (input.hubSignature && input.rawBody && signatureMatches(secret, input.rawBody, input.hubSignature.trim())) {
+      return { ok: true, credential: { kind: 'github-signature' } };
+    }
+  }
+
+  // An admin's own session, carried as the bearer. Checked after the secret
+  // so a secret that happens to parse as a JWT is never mistaken for one.
+  if (bearer) {
+    const claim = session.verifyJwt(bearer);
+    if (claim && (await session.isAdmin(claim.email))) {
+      return { ok: true, credential: { kind: 'admin-session', email: claim.email } };
+    }
+  }
+
+  // Nothing matched. Say WHY when the reason is the deployment's, not the
+  // caller's: a hook pointed at an unconfigured endpoint should read "set
+  // the secret", not "wrong secret".
+  if (!secret && (bearer || input.gitlabToken || input.hubSignature)) {
+    return { ok: false, status: 503, message: configured ? SECRET_TOO_SHORT : NO_SECRET };
+  }
+  return {
+    ok: false,
+    status: 401,
+    message: 'A valid sync secret or an administrator session is required.',
+  };
+}

package/src/modules/kb-sync/sync-payload.ts

@@ -0,0 +1,143 @@
+import { assertValidBranchName } from '../kb-fs/branch-name.js';
+
+/**
+ * Turn whatever a caller POSTed to `/api/sync` into the branches to sync.
+ *
+ * Pure: no IO, no throwing on foreign shapes. The endpoint is meant to be
+ * pointed at by a webhook subscription whose payload the operator does not
+ * control, so an unrecognised body must degrade to "sync everything" — the
+ * safe answer, and the one an empty pipeline call gets too — never to a 400
+ * that makes the host disable the subscription.
+ *
+ * Recognised shapes:
+ *
+ *   explicit      `{ "branches": ["main", "ali/x"] }`
+ *   azure-devops  `eventType: git.push` → `resource.refUpdates[].name`
+ *                 `eventType: git.pullrequest.*` → `resource.targetRefName`
+ *                 and `resource.sourceRefName`
+ *   github/gitea  `{ "ref": "refs/heads/main" }` (push)
+ *   gitlab        `{ "object_kind": "push", "ref": "refs/heads/main" }`
+ *
+ * Only `refs/heads/*` refs count; a tag push names nothing to sync. A branch
+ * name git would refuse is dropped from a provider payload (the host
+ * produced it, nothing we can do) but reported for an explicit body, where
+ * it is the caller's mistake to fix.
+ */
+export type SyncPayloadSource = 'explicit' | 'azure-devops' | 'github' | 'gitlab' | 'none';
+
+export interface ParsedSyncPayload {
+  source: SyncPayloadSource;
+  branches: string[] | 'all';
+  /** Explicit-body names git would refuse. Empty for every other source. */
+  invalid: string[];
+}
+
+const HEADS = 'refs/heads/';
+
+function isRecord(v: unknown): v is Record<string, unknown> {
+  return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+
+function branchFromRef(ref: unknown): string | null {
+  if (typeof ref !== 'string' || !ref.startsWith(HEADS)) return null;
+  const name = ref.slice(HEADS.length);
+  return isValidBranch(name) ? name : null;
+}
+
+function isValidBranch(name: string): boolean {
+  try {
+    assertValidBranchName(name);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function unique(names: string[]): string[] {
+  return [...new Set(names)];
+}
+
+/**
+ * How an invalid explicit entry is echoed back. Only primitives are
+ * stringified: `String(obj)` on a JSON object carrying `"toString": 1` throws
+ * ("Cannot convert object to primitive value"), which would turn a caller's
+ * bad body into our 500.
+ */
+function describeInvalid(entry: unknown): string {
+  if (entry === null) return 'null';
+  switch (typeof entry) {
+    case 'string':
+      return entry.trim() === '' ? '(empty)' : entry;
+    case 'number':
+    case 'boolean':
+    case 'bigint':
+      return String(entry);
+    default:
+      return Array.isArray(entry) ? '(array)' : '(object)';
+  }
+}
+
+export function parseSyncPayload(body: unknown): ParsedSyncPayload {
+  if (!isRecord(body)) return { source: 'none', branches: 'all', invalid: [] };
+
+  if ('branches' in body) {
+    const raw = body.branches;
+    if (!Array.isArray(raw)) return { source: 'explicit', branches: 'all', invalid: [] };
+    const valid: string[] = [];
+    const invalid: string[] = [];
+    for (const entry of raw) {
+      if (typeof entry !== 'string' || !entry.trim()) {
+        invalid.push(describeInvalid(entry));
+        continue;
+      }
+      const name = entry.trim();
+      (isValidBranch(name) ? valid : invalid).push(name);
+    }
+    return { source: 'explicit', branches: unique(valid), invalid };
+  }
+
+  // Azure DevOps service hook. `eventType` names the event; the branches sit
+  // under `resource` in a per-event shape.
+  const eventType = body.eventType;
+  if (typeof eventType === 'string' && isRecord(body.resource)) {
+    const resource = body.resource;
+    if (eventType === 'git.push') {
+      const updates = Array.isArray(resource.refUpdates) ? resource.refUpdates : [];
+      // A DELETION (new object id all zeros) names its branch too: syncing it
+      // is what finds the branch gone and retires the stale clone, instead of
+      // leaving it to fail on some unrelated full sync later.
+      const branches = updates
+        .map((u) => (isRecord(u) ? branchFromRef(u.name) : null))
+        .filter((b): b is string => b !== null);
+      return { source: 'azure-devops', branches: unique(branches), invalid: [] };
+    }
+    if (eventType.startsWith('git.pullrequest.')) {
+      const branches = [branchFromRef(resource.targetRefName), branchFromRef(resource.sourceRefName)]
+        .filter((b): b is string => b !== null);
+      return { source: 'azure-devops', branches: unique(branches), invalid: [] };
+    }
+  }
+
+  // GitLab events all carry `object_kind`. Only a push (or a tag push, which
+  // names no branch) is a statement about a ref; anything else with a stray
+  // `ref` — a merge request, a pipeline — is not, and takes the "everything"
+  // fallback rather than being read as a push of that ref.
+  if (typeof body.object_kind === 'string') {
+    if (body.object_kind !== 'push' && body.object_kind !== 'tag_push') {
+      return { source: 'none', branches: 'all', invalid: [] };
+    }
+    // A deletion (a push to the null sha) names its branch like any push: the
+    // sync finds it gone and retires the clone.
+    const branch = branchFromRef(body.ref);
+    return { source: 'gitlab', branches: branch ? [branch] : [], invalid: [] };
+  }
+
+  // GitHub / Gitea push: `ref` at the top level, no `object_kind`. A deletion
+  // (`deleted: true`) names its branch like any push, for the same reason.
+  if (typeof body.ref === 'string') {
+    const branch = branchFromRef(body.ref);
+    return { source: 'github', branches: branch ? [branch] : [], invalid: [] };
+  }
+
+  return { source: 'none', branches: 'all', invalid: [] };
+}