@bevel-software/platform-core-backend 0.14.0 → 0.19.0

package/src/modules/kb-sync/__tests__/sync-auth.test.ts

@@ -0,0 +1,118 @@
+import { createHmac } from 'node:crypto';
+import { describe, expect, it, vi } from 'vitest';
+import { verifySyncCredential, type SyncSessionVerifier } from '../sync-auth.js';
+
+const SECRET = 'a-very-long-and-random-sync-secret';
+
+function session(opts: { email?: string; admin?: boolean } = {}): SyncSessionVerifier {
+  return {
+    verifyJwt: vi.fn((token: string) =>
+      token === 'jwt-ok' && opts.email ? { email: opts.email } : null,
+    ),
+    isAdmin: vi.fn(async () => opts.admin === true),
+  };
+}
+
+describe('verifySyncCredential', () => {
+  it('accepts the secret as a bearer', async () => {
+    const r = await verifySyncCredential(
+      { secret: SECRET, authorization: `Bearer ${SECRET}` },
+      session(),
+    );
+    expect(r).toEqual({ ok: true, credential: { kind: 'bearer' } });
+  });
+
+  it('refuses a wrong bearer with 401', async () => {
+    const r = await verifySyncCredential(
+      { secret: SECRET, authorization: 'Bearer nope' },
+      session(),
+    );
+    expect(r).toMatchObject({ ok: false, status: 401 });
+  });
+
+  it('refuses a missing credential with 401', async () => {
+    const r = await verifySyncCredential({ secret: SECRET }, session());
+    expect(r).toMatchObject({ ok: false, status: 401 });
+  });
+
+  it('accepts the GitLab token header', async () => {
+    const r = await verifySyncCredential({ secret: SECRET, gitlabToken: SECRET }, session());
+    expect(r).toEqual({ ok: true, credential: { kind: 'gitlab-token' } });
+  });
+
+  it('accepts a GitHub signature over the raw body, and only over those bytes', async () => {
+    const body = Buffer.from('{"ref":"refs/heads/main"}');
+    const sig = 'sha256=' + createHmac('sha256', SECRET).update(body).digest('hex');
+    expect(
+      await verifySyncCredential({ secret: SECRET, hubSignature: sig, rawBody: body }, session()),
+    ).toEqual({ ok: true, credential: { kind: 'github-signature' } });
+    expect(
+      await verifySyncCredential(
+        { secret: SECRET, hubSignature: sig, rawBody: Buffer.from('{"ref":"refs/heads/other"}') },
+        session(),
+      ),
+    ).toMatchObject({ ok: false, status: 401 });
+    expect(
+      await verifySyncCredential(
+        { secret: SECRET, hubSignature: 'sha1=abc', rawBody: body },
+        session(),
+      ),
+    ).toMatchObject({ ok: false, status: 401 });
+  });
+
+  it("accepts an admin's own session as the bearer", async () => {
+    const r = await verifySyncCredential(
+      { secret: SECRET, authorization: 'Bearer jwt-ok' },
+      session({ email: 'admin@example.com', admin: true }),
+    );
+    expect(r).toEqual({
+      ok: true,
+      credential: { kind: 'admin-session', email: 'admin@example.com' },
+    });
+  });
+
+  it("refuses a non-admin's session", async () => {
+    const r = await verifySyncCredential(
+      { secret: SECRET, authorization: 'Bearer jwt-ok' },
+      session({ email: 'someone@example.com', admin: false }),
+    );
+    expect(r).toMatchObject({ ok: false, status: 401 });
+  });
+
+  it('with no secret configured, a header credential gets 503 that says so', async () => {
+    const r = await verifySyncCredential({ secret: '', authorization: 'Bearer anything' }, session());
+    expect(r).toMatchObject({ ok: false, status: 503 });
+    expect((r as { message: string }).message).toContain('KB_SYNC_SECRET');
+  });
+
+  it('with no secret configured, an admin session still works', async () => {
+    const r = await verifySyncCredential(
+      { secret: '', authorization: 'Bearer jwt-ok' },
+      session({ email: 'admin@example.com', admin: true }),
+    );
+    expect(r).toMatchObject({ ok: true });
+  });
+
+  it('with no secret configured and no credential, 401 not 503', async () => {
+    expect(await verifySyncCredential({ secret: '' }, session())).toMatchObject({
+      ok: false,
+      status: 401,
+    });
+  });
+});
+
+describe('verifySyncCredential — a secret set too short', () => {
+  it('is refused with 503 that says so, even when the caller presents it exactly', async () => {
+    const r = await verifySyncCredential({ secret: 'short', authorization: 'Bearer short' }, session());
+    expect(r).toMatchObject({ ok: false, status: 503 });
+    expect((r as { message: string }).message).toContain('shorter than 16');
+  });
+
+  it('an admin session still works past a too-short secret', async () => {
+    const r = await verifySyncCredential(
+      { secret: 'short', authorization: 'Bearer jwt-ok' },
+      session({ email: 'admin@example.com', admin: true }),
+    );
+    expect(r).toMatchObject({ ok: true });
+  });
+});

package/src/modules/kb-sync/__tests__/sync-payload.test.ts

@@ -0,0 +1,148 @@
+import { describe, expect, it } from 'vitest';
+import { parseSyncPayload } from '../sync-payload.js';
+
+describe('parseSyncPayload', () => {
+  it('an empty or foreign body syncs everything', () => {
+    expect(parseSyncPayload(undefined)).toEqual({ source: 'none', branches: 'all', invalid: [] });
+    expect(parseSyncPayload({ hello: 'world' })).toEqual({ source: 'none', branches: 'all', invalid: [] });
+    expect(parseSyncPayload('text')).toEqual({ source: 'none', branches: 'all', invalid: [] });
+  });
+
+  it('explicit branches are kept in order, trimmed and de-duplicated', () => {
+    expect(parseSyncPayload({ branches: [' main ', 'ali/x', 'main'] })).toEqual({
+      source: 'explicit',
+      branches: ['main', 'ali/x'],
+      invalid: [],
+    });
+  });
+
+  it('an explicit name git would refuse is reported, not silently dropped', () => {
+    const parsed = parseSyncPayload({ branches: ['main', '--upload-pack=x', 'a..b', 7] });
+    expect(parsed.branches).toEqual(['main']);
+    expect(parsed.invalid).toEqual(['--upload-pack=x', 'a..b', '7']);
+  });
+
+  it('an explicit `branches` that is not a list means everything', () => {
+    expect(parseSyncPayload({ branches: 'main' }).branches).toBe('all');
+  });
+
+  it('Azure DevOps push: every updated head ref, tags ignored', () => {
+    const parsed = parseSyncPayload({
+      eventType: 'git.push',
+      resource: {
+        refUpdates: [
+          { name: 'refs/heads/main', oldObjectId: 'a', newObjectId: 'b' },
+          { name: 'refs/tags/v1', oldObjectId: '0', newObjectId: 'c' },
+          { name: 'refs/heads/ali/new-skill' },
+        ],
+      },
+    });
+    expect(parsed).toEqual({
+      source: 'azure-devops',
+      branches: ['main', 'ali/new-skill'],
+      invalid: [],
+    });
+  });
+
+  it('Azure DevOps pull request: target then source', () => {
+    for (const eventType of ['git.pullrequest.created', 'git.pullrequest.updated', 'git.pullrequest.merged']) {
+      const parsed = parseSyncPayload({
+        eventType,
+        resource: {
+          sourceRefName: 'refs/heads/ali/new-skill',
+          targetRefName: 'refs/heads/main',
+          status: 'completed',
+        },
+      });
+      expect(parsed).toEqual({
+        source: 'azure-devops',
+        branches: ['main', 'ali/new-skill'],
+        invalid: [],
+      });
+    }
+  });
+
+  it('an Azure DevOps event we do not know syncs everything', () => {
+    expect(parseSyncPayload({ eventType: 'build.complete', resource: {} }).branches).toBe('all');
+  });
+
+  it('GitHub push: the pushed branch', () => {
+    expect(parseSyncPayload({ ref: 'refs/heads/main', before: 'a', after: 'b' })).toEqual({
+      source: 'github',
+      branches: ['main'],
+      invalid: [],
+    });
+  });
+
+  it('GitHub tag push names nothing to sync', () => {
+    expect(parseSyncPayload({ ref: 'refs/tags/v1' })).toEqual({
+      source: 'github',
+      branches: [],
+      invalid: [],
+    });
+  });
+
+  it('GitLab push: the pushed branch', () => {
+    expect(parseSyncPayload({ object_kind: 'push', ref: 'refs/heads/main' })).toEqual({
+      source: 'gitlab',
+      branches: ['main'],
+      invalid: [],
+    });
+  });
+});
+
+describe('parseSyncPayload — hostile shapes', () => {
+  it('an explicit entry that cannot be stringified is reported, not thrown', () => {
+    const parsed = parseSyncPayload({ branches: ['main', { toString: 1 }, [1], null] });
+    expect(parsed.branches).toEqual(['main']);
+    expect(parsed.invalid).toEqual(['(object)', '(array)', 'null']);
+  });
+
+  it('a GitLab event that is not a push falls back to everything, even with a stray ref', () => {
+    expect(parseSyncPayload({ object_kind: 'merge_request', ref: 'refs/heads/main' })).toEqual({
+      source: 'none',
+      branches: 'all',
+      invalid: [],
+    });
+  });
+
+  it('a GitLab tag push names nothing to sync', () => {
+    expect(parseSyncPayload({ object_kind: 'tag_push', ref: 'refs/tags/v1' })).toEqual({
+      source: 'gitlab',
+      branches: [],
+      invalid: [],
+    });
+  });
+});
+
+describe('parseSyncPayload — deletions', () => {
+  it('a GitHub branch deletion names its branch, so the sync finds it gone and retires the clone', () => {
+    expect(parseSyncPayload({ ref: 'refs/heads/ali/x', deleted: true, after: '0'.repeat(40) })).toEqual({
+      source: 'github',
+      branches: ['ali/x'],
+      invalid: [],
+    });
+  });
+
+  it('an Azure DevOps deletion (new object id all zeros) names its branch beside the real update', () => {
+    expect(
+      parseSyncPayload({
+        eventType: 'git.push',
+        resource: {
+          refUpdates: [
+            { name: 'refs/heads/ali/x', oldObjectId: 'a'.repeat(40), newObjectId: '0'.repeat(40) },
+            { name: 'refs/heads/main', oldObjectId: 'b'.repeat(40), newObjectId: 'c'.repeat(40) },
+          ],
+        },
+      }),
+    ).toEqual({ source: 'azure-devops', branches: ['ali/x', 'main'], invalid: [] });
+  });
+
+  it('a GitLab branch deletion (after all zeros) names its branch', () => {
+    expect(parseSyncPayload({ object_kind: 'push', ref: 'refs/heads/ali/x', after: '0'.repeat(40) })).toEqual({
+      source: 'gitlab',
+      branches: ['ali/x'],
+      invalid: [],
+    });
+  });
+});

package/src/modules/kb-sync/index.ts

@@ -0,0 +1,19 @@
+export type {
+  IKbSyncService,
+  LastSync,
+  SyncRequest,
+  SyncResult,
+  SyncWorkflowPort,
+  SyncWorkspacePort,
+} from './kb-sync.interface.js';
+export { KbSyncService } from './kb-sync.service.js';
+export {
+  createKbSyncRoutes,
+  httpStatusFor,
+  isSyncRawBodyPath,
+  SYNC_RESPONSE_HEADER,
+  SYNC_RESPONSE_MARKER,
+  type KbSyncRouteDeps,
+} from './kb-sync.routes.js';
+export { parseSyncPayload, type ParsedSyncPayload, type SyncPayloadSource } from './sync-payload.js';
+export { verifySyncCredential, type SyncAuthInput, type SyncAuthResult, type SyncCredential } from './sync-auth.js';

package/src/modules/kb-sync/kb-sync.interface.ts

@@ -0,0 +1,79 @@
+import type { BranchSyncOutcome } from '@bevel-software/platform-shared';
+
+/**
+ * Remote sync: a git host (Azure DevOps, GitHub, GitLab, a pipeline) tells
+ * Hexis "the repository changed" and Hexis brings the clones it holds up to
+ * date. The module owns the HTTP surface, the credential check, the payload
+ * parsing and the coalescing; every git operation happens behind the
+ * workflow port below, never here.
+ */
+
+/** Which branches a caller asked to sync. `'all'` = every clone Hexis holds. */
+export interface SyncRequest {
+  branches: string[] | 'all';
+  /** Who asked — a credential kind or an admin's email — for the last-sync record. */
+  by?: string;
+}
+
+/**
+ * What the most recent sync did, for the admin's Deployment page. Kept in
+ * memory only: it answers "is the hook wired up and working?", and a restart
+ * resetting it to "none yet" is the honest answer for a fresh process.
+ */
+export interface LastSync {
+  /** Epoch ms when the sync finished. */
+  at: number;
+  by: string;
+  status: SyncResult['status'];
+  /** One entry per branch, as the response carried them. */
+  results: BranchSyncOutcome[];
+}
+
+export interface SyncResult {
+  /** `synced` when no branch needs a person or a retry; `partial` otherwise. */
+  status: 'synced' | 'partial';
+  results: BranchSyncOutcome[];
+  changeRequests: {
+    /** Open change requests closed because one of their branches is gone from the host. */
+    closedDeletedBranch: number;
+  };
+}
+
+export interface IKbSyncService {
+  /**
+   * Sync the requested branches. One sync runs at a time; a request arriving
+   * mid-run is folded into a single queued follow-up (the union of every
+   * request that arrived meanwhile) and resolves with that follow-up's result.
+   * Never rejects for a per-branch failure — those are outcomes.
+   */
+  sync(request: SyncRequest): Promise<SyncResult>;
+  /** The most recent completed sync in this process, or null before the first. */
+  lastSync(): LastSync | null;
+}
+
+/** The slice of the workflow module a sync drives. */
+export interface SyncWorkflowPort {
+  syncWorkspaceFromRemote(workspaceId: string): Promise<BranchSyncOutcome>;
+  /**
+   * Retire the clone of a branch the host deleted, under the branch-lifecycle
+   * lock and only if origin still lacks it. Returns whether it was removed.
+   */
+  retireRemoteGoneClone(workspaceId: string): Promise<boolean>;
+  /** The existing sweep; returns how many requests it closed. */
+  closeChangeRequestsWithDeletedBranches(): Promise<number>;
+}
+
+/**
+ * The slice of the workspace module a sync reads: which clones exist ON
+ * DISK. Not the in-memory "seen this process" cache — clones survive a
+ * restart and a hook that fires before anyone has opened a branch must still
+ * find them.
+ */
+export interface SyncWorkspacePort {
+  /**
+   * `unreadable` names a clone the listing could see but not probe (an ACL,
+   * a half-deleted directory); the sync reports it as that branch's error
+   * rather than letting one bad directory fail every other branch.
+   */
+  listClonedWorkspaces(): Promise<Array<{ id: string; branch: string; unreadable?: string }>>;
+}

package/src/modules/kb-sync/kb-sync.routes.ts

@@ -0,0 +1,200 @@
+import express from 'express';
+import { logger } from '../../shared/logging.js';
+
+const log = logger('sync');
+import type { IAdminAccessService } from '../admin/admin.interface.js';
+import { assertValidBranchName } from '../kb-fs/branch-name.js';
+import type { IKbSyncService, SyncResult } from './kb-sync.interface.js';
+import { parseSyncPayload, type SyncPayloadSource } from './sync-payload.js';
+import { verifySyncCredential } from './sync-auth.js';
+
+/**
+ * Remote sync — a git host or a pipeline tells Hexis the repository changed.
+ *
+ *   POST /api/sync/<branch>   sync that one branch — the branch whose change
+ *                             made the call. What a GitHub Action or a
+ *                             pipeline step uses: the branch name goes in the
+ *                             URL, the body is ignored for branch selection.
+ *   POST /api/sync            the branches named by the body — an Azure
+ *                             DevOps service hook or a GitHub/GitLab webhook
+ *                             payload (they cannot template the URL), an
+ *                             explicit `{ "branches": [...] }`, or nothing,
+ *                             which syncs every clone Hexis holds.
+ *
+ * Mounted BEFORE every JWT-protected `/api` mount: the caller is a machine
+ * holding the deployment's sync secret, not a browser session, and Express 5
+ * would otherwise 401 it in the first protected mount. The route parses its
+ * own body as raw bytes (the global JSON parser skips this path) because the
+ * GitHub-style credential is an HMAC over exactly the bytes that arrived.
+ *
+ * Status codes are chosen for the two kinds of caller. A pipeline's
+ * `curl --fail` must fail exactly when Hexis is not in sync; a webhook host
+ * retries 5xx and gives up on 4xx:
+ *
+ *   200  every branch updated, already current, or not cloned here
+ *   409  a branch has a CONFLICT: Hexis-side commits clash with the host.
+ *        Automatic recovery is queued, but the branch is not in sync at the
+ *        time of the answer; the body's `error` names the files. A retry
+ *        answers 200 once recovery (or a person) has cleared it.
+ *   503  a branch could not be pulled (origin unreachable, credential
+ *        refused) — retriable. Also: no sync secret configured.
+ *   401  missing or wrong credential
+ *   400  a branch name git would refuse, or a body that is not JSON
+ */
+export interface KbSyncRouteDeps {
+  kbSync: IKbSyncService;
+  /** The configured secret, read per request so a setup-screen save applies at once. */
+  syncSecret: () => string;
+  authService: { verifyToken(token: string): { userId: string; email: string } };
+  adminAccess: IAdminAccessService;
+}
+
+/** Cap on a webhook body. ADO's PR payloads run to a few hundred KB. */
+const BODY_LIMIT = '2mb';
+
+/**
+ * Stamped on EVERY response this router writes. The endpoint answers 503 for
+ * an ordinary result (a branch could not be pulled), and a reverse proxy
+ * answers 503 when the backend is down; by status alone the browser cannot
+ * tell them apart, and it must — one is a line on the page, the other the
+ * maintenance overlay. A proxy standing in for a dead upstream never sets
+ * this header. Exposed to cross-origin callers in `create-core-server.ts`.
+ */
+export const SYNC_RESPONSE_HEADER = 'X-Hexis-Sync';
+export const SYNC_RESPONSE_MARKER = 'result';
+
+/**
+ * Whether a request path belongs to this router and must reach it with its
+ * body UNPARSED. The server's global JSON parser consults this: once that
+ * parser has read the stream, `express.raw` below sees nothing and the
+ * signature credential has no bytes to verify. Both routes share the check,
+ * so the branch form (`/api/sync/<branch>`) is covered as well as the bare
+ * one — an exact-path exemption used to protect only the latter.
+ */
+export function isSyncRawBodyPath(path: string): boolean {
+  // Case-insensitive, because Express routing is: `/api/Sync` reaches this
+  // router, and if this check said no the global parser would eat the body
+  // first. On either form a signed call would then 401 (no bytes to verify).
+  // On the bare form a bearer call would silently sync every clone, since a
+  // body that is no longer a Buffer parses as "no body"; the branch form is
+  // unaffected there, its branch comes from the URL.
+  const lower = path.toLowerCase();
+  return lower === '/api/sync' || lower.startsWith('/api/sync/');
+}
+
+export function httpStatusFor(result: SyncResult): 200 | 409 | 503 {
+  if (result.results.some((r) => r.outcome === 'conflict')) return 409;
+  if (result.results.some((r) => r.outcome === 'error')) return 503;
+  return 200;
+}
+
+type Selection = { source: SyncPayloadSource | 'path'; branches: string[] | 'all' };
+
+export function createKbSyncRoutes(deps: KbSyncRouteDeps): express.Router {
+  const router = express.Router();
+
+  const rawBody = express.raw({ type: () => true, limit: BODY_LIMIT });
+
+  /**
+   * Everything both routes share: the credential check, then the sync and
+   * its answer. `select` turns the request into the branches to sync, or
+   * sends its own 400 and returns null.
+   */
+  async function handle(
+    req: express.Request,
+    res: express.Response,
+    select: () => Selection | null,
+  ): Promise<void> {
+    res.setHeader(SYNC_RESPONSE_HEADER, SYNC_RESPONSE_MARKER);
+    const raw: Buffer | undefined = Buffer.isBuffer(req.body) ? req.body : undefined;
+
+    const auth = await verifySyncCredential(
+      {
+        secret: deps.syncSecret(),
+        authorization: header(req, 'authorization'),
+        gitlabToken: header(req, 'x-gitlab-token'),
+        hubSignature: header(req, 'x-hub-signature-256'),
+        rawBody: raw,
+      },
+      {
+        verifyJwt: (token) => {
+          try {
+            return { email: deps.authService.verifyToken(token).email };
+          } catch {
+            return null;
+          }
+        },
+        isAdmin: (email) => deps.adminAccess.isAdmin(email),
+      },
+    );
+    if (!auth.ok) {
+      if (auth.status === 401) res.setHeader('WWW-Authenticate', 'Bearer realm="hexis-sync"');
+      res.status(auth.status).json({ error: auth.message });
+      return;
+    }
+
+    const selection = select();
+    if (!selection) return;
+
+    const who = auth.credential.kind === 'admin-session' ? auth.credential.email : auth.credential.kind;
+    const asked =
+      selection.branches === 'all' ? 'all cloned branches' : selection.branches.join(', ') || '(none)';
+    try {
+      const result = await deps.kbSync.sync({ branches: selection.branches, by: who });
+      const summary = result.results.map((r) => `${r.branch}=${r.outcome}`).join(' ');
+      log.info(`by ${who} via ${selection.source} for ${asked}: ${result.status} ${summary}`);
+      res.status(httpStatusFor(result)).json(result);
+    } catch (err) {
+      log.error('unhandled error:', { err });
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  // The branch in the URL. A wildcard rather than `:branch` so a branch with
+  // slashes (`ali/new-skill`) works spelled either way — as segments or
+  // percent-encoded as one.
+  router.post('/sync/*branch', rawBody, (req, res) =>
+    handle(req, res, () => {
+      const param = req.params.branch as unknown;
+      const branch = (Array.isArray(param) ? param.join('/') : String(param ?? '')).trim();
+      try {
+        assertValidBranchName(branch);
+      } catch {
+        res.status(400).json({ error: `Not a valid branch name: ${branch}` });
+        return null;
+      }
+      return { source: 'path', branches: [branch] };
+    }),
+  );
+
+  router.post('/sync', rawBody, (req, res) =>
+    handle(req, res, () => {
+      const raw: Buffer | undefined = Buffer.isBuffer(req.body) ? req.body : undefined;
+      let body: unknown = undefined;
+      if (raw && raw.length > 0) {
+        try {
+          body = JSON.parse(raw.toString('utf8'));
+        } catch {
+          res.status(400).json({ error: 'The request body is not valid JSON.' });
+          return null;
+        }
+      }
+      const parsed = parseSyncPayload(body);
+      if (parsed.invalid.length > 0) {
+        res.status(400).json({
+          error: `Not a valid branch name: ${parsed.invalid.join(', ')}`,
+          invalid: parsed.invalid,
+        });
+        return null;
+      }
+      return { source: parsed.source, branches: parsed.branches };
+    }),
+  );
+
+  return router;
+}
+
+function header(req: express.Request, name: string): string | undefined {
+  const v = req.headers[name];
+  return typeof v === 'string' ? v : undefined;
+}