@bevel-software/platform-core-backend 0.20.0 → 0.22.0

package/src/modules/settings/deployment-settings.service.ts

@@ -17,6 +17,7 @@ import { TokenCrypto } from '../../shared/token-crypto.js';
 import { assertKbDirNameFree } from '../kb-fs/repo-path.js';
 import { parseRetentionWindow } from '../audit/audit.contract.js';
 import { normalizeIssuerUrl } from './oidc-check.js';
+import { GIT_MODES, isGitMode } from './repository-source.js';
 
 /**
  * A setting an admin may set from the setup screen instead of the environment.
@@ -69,6 +70,17 @@ export interface SettingDef {
    * restart for a change that never happened.
    */
   unsetMeans?: string;
+  /**
+   * Written by the deployment itself, at the end of a flow that PROVED the
+   * value, and never taken from a save. What a GitHub App was given when it
+   * was registered, and which installation of it the admin was shown to
+   * hold, are facts somebody established; accepted from a form they would
+   * be claims, and the claim "installation 42 is mine" is one that reaches
+   * another organisation's repositories. A save that names one is refused,
+   * the setup screen is not told about it, and its value is read like any
+   * other's (the environment first), so an operator can still pin it.
+   */
+  internal?: boolean;
 }
 
 /**
@@ -99,6 +111,25 @@ export const validateHttpsRemote = (value: string): string | null => {
  * The core catalogue. Order is the order the setup screen renders them in.
  */
 export const CORE_SETTINGS: SettingDef[] = [
+  {
+    /**
+     * Which way the deployment is given its repository (see
+     * `RepositorySource`). Unset on a deployment configured before there was
+     * a choice, which is read as `token` when it has an address or a token,
+     * and as nothing chosen when it has neither.
+     *
+     * Restart-to-apply on a deployment that is running: its working copies
+     * were cloned from the repository it had, and they are brought into line
+     * with another one by the startup phase, at a moment when nobody is
+     * using them. The save that completes first-run setup runs that phase
+     * itself, and owes no restart.
+     */
+    key: 'gitMode',
+    envVar: 'GIT_MODE',
+    section: 'knowledge-base',
+    validate: (v) => (isGitMode(v) ? null : `Choose one of: ${GIT_MODES.join(', ')}.`),
+    restartToApply: true,
+  },
   {
     key: 'kbRepoUrl',
     envVar: 'KB_REPO_URL',
@@ -121,6 +152,59 @@ export const CORE_SETTINGS: SettingDef[] = [
     validate: (v) =>
       /^[A-Za-z0-9._-]+$/.test(v) ? null : 'Use only letters, digits, dot, underscore or hyphen.',
   },
+
+  /**
+   * A repository on GitHub, reached through a GitHub App (see
+   * `modules/github-app/`).
+   *
+   * THE REPOSITORY is the admin's to choose, among those the installation
+   * reaches, and is saved like any setting: a name the installation does not
+   * reach fails the connection check, since the only token presented to
+   * GitHub is the installation's.
+   *
+   * EVERYTHING ELSE is internal: the app's identity and keys, as GitHub
+   * issued them when the app was registered, and the installation, as
+   * GitHub confirmed the admin holds it. Each has a variable, so whoever
+   * operates the deployment can supply an app they registered themselves,
+   * and a host serving many deployments can supply one app for all of them.
+   */
+  {
+    key: 'githubRepository',
+    envVar: 'GITHUB_APP_REPOSITORY',
+    section: 'knowledge-base',
+    validate: (v) =>
+      /^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?\/[A-Za-z0-9._-]+$/.test(v) && !v.endsWith('.git')
+        ? null
+        : 'Name the repository as owner/name.',
+  },
+  { key: 'githubAppId', envVar: 'GITHUB_APP_ID', section: 'knowledge-base', internal: true, validate: (v) => (/^\d+$/.test(v) ? null : 'The app id is a number.') },
+  { key: 'githubAppSlug', envVar: 'GITHUB_APP_SLUG', section: 'knowledge-base', internal: true, validate: (v) => (/^[A-Za-z0-9-]+$/.test(v) ? null : 'The app slug is letters, digits and hyphens.') },
+  { key: 'githubAppPrivateKey', envVar: 'GITHUB_APP_PRIVATE_KEY', section: 'knowledge-base', internal: true, secret: true },
+  { key: 'githubAppClientId', envVar: 'GITHUB_APP_CLIENT_ID', section: 'knowledge-base', internal: true },
+  { key: 'githubAppClientSecret', envVar: 'GITHUB_APP_CLIENT_SECRET', section: 'knowledge-base', internal: true, secret: true },
+  {
+    /**
+     * Put in front of the state every round trip to GitHub carries, by a
+     * host that serves several deployments behind ONE app: GitHub sends the
+     * browser back to the one address registered for the app, and the host
+     * reads this to know which deployment to hand it to. Never stored.
+     */
+    key: 'githubAppStateTag',
+    envVar: 'GITHUB_APP_STATE_TAG',
+    section: 'knowledge-base',
+    internal: true,
+    validate: (v) => (/^[a-z0-9-]{1,63}$/.test(v) ? null : 'Lowercase letters, digits and hyphens.'),
+  },
+  { key: 'githubInstallationId', envVar: 'GITHUB_APP_INSTALLATION_ID', section: 'knowledge-base', internal: true, validate: (v) => (/^\d+$/.test(v) ? null : 'The installation id is a number.') },
+  /** Whose account the installation is on, as GitHub names it: shown, never trusted. */
+  { key: 'githubInstallationAccount', section: 'knowledge-base', internal: true },
+  /**
+   * The repositories the person who connected the installation could PUSH
+   * to with their own GitHub account, one `owner/name` a line, as GitHub
+   * listed them when they connected. The deployment may be pointed at one
+   * of these and at nothing else the installation reaches.
+   */
+  { key: 'githubRepositoriesPermitted', section: 'knowledge-base', internal: true },
   {
     key: 'kbDirName',
     envVar: 'KB_DIR_NAME',
@@ -181,7 +265,9 @@ export const CORE_SETTINGS: SettingDef[] = [
      */
     key: 'agentsFile',
     section: 'knowledge-base',
-    validate: (v) => validateAgentsFileName(v),
+    // Judged against the default roots here; the quartet check in `plan`
+    // judges it against the roots the same save puts in effect.
+    validate: (v) => validateAgentsFileName(v, DEFAULT_KB_LAYOUT),
     restartToApply: true,
     unsetMeans: DEFAULT_KB_LAYOUT.agentsFile,
   },
@@ -422,11 +508,23 @@ export class DeploymentSettingsService {
   private stored = new Map<string, string>();
   private readonly crypto: TokenCrypto | null;
 
+  /**
+   * The environment the env-first layer reads. The process's own for a
+   * single-tenant deployment; a host serving several knowledge bases hands
+   * each graph one built from its tenant record, so a variable set on the
+   * host process can never leak into every tenant, and a record's values
+   * behave exactly as environment-pinned ones do (they win over the setup
+   * screen and the screen shows them as such).
+   */
+  private readonly env: NodeJS.ProcessEnv;
+
   constructor(
     private readonly db: Database,
     private readonly secretsEncKey: string,
     defs: SettingDef[] = CORE_SETTINGS,
+    options: { env?: NodeJS.ProcessEnv } = {},
   ) {
+    this.env = options.env ?? process.env;
     for (const def of defs) this.defs.set(def.key, def);
     // No key configured means secrets cannot be stored — surfaced when someone
     // tries, rather than pretended away by writing plaintext.
@@ -470,7 +568,7 @@ export class DeploymentSettingsService {
   resolve(key: string): string {
     const def = this.defs.get(key);
     if (!def) return '';
-    const fromEnv = def.envVar ? (process.env[def.envVar] ?? '').trim() : '';
+    const fromEnv = def.envVar ? (this.env[def.envVar] ?? '').trim() : '';
     if (fromEnv) return fromEnv;
     return (this.stored.get(key) ?? '').trim();
   }
@@ -521,7 +619,7 @@ export class DeploymentSettingsService {
    */
   async importLegacyLayoutEnv(): Promise<void> {
     for (const [key, envVar] of Object.entries(LEGACY_LAYOUT_ENV_VARS)) {
-      const fromEnv = (process.env[envVar] ?? '').trim();
+      const fromEnv = (this.env[envVar] ?? '').trim();
       if (!fromEnv) continue;
       const saved = (this.stored.get(key) ?? '').trim();
       if (saved) {
@@ -559,7 +657,7 @@ export class DeploymentSettingsService {
     // A setting with no variable can never read `env`, whatever the process
     // environment happens to hold — which is what makes the layout fields
     // editable in the app on a deployment that still sets the old variables.
-    if (def.envVar && (process.env[def.envVar] ?? '').trim()) return 'env';
+    if (def.envVar && (this.env[def.envVar] ?? '').trim()) return 'env';
     return (this.stored.get(key) ?? '').trim() ? 'stored' : 'unset';
   }
 
@@ -569,7 +667,7 @@ export class DeploymentSettingsService {
    * replace it, which is all anyone needs to finish setup.
    */
   describe(): ResolvedSetting[] {
-    return this.definitions.map((def) => {
+    return this.definitions.filter((def) => !def.internal).map((def) => {
       const source = this.sourceOf(def.key);
       const base = {
         key: def.key,
@@ -602,8 +700,10 @@ export class DeploymentSettingsService {
   async save(
     entries: Record<string, string>,
     updatedBy: string | null,
+    /** Set by {@link record} alone: the batch is the deployment's own, and may name internal settings. */
+    by: 'admin' | 'deployment' = 'admin',
   ): Promise<{ restartRequired: boolean; restartKeys: string[] }> {
-    const { toWrite, toClear } = this.plan(entries);
+    const { toWrite, toClear } = this.plan(entries, by);
 
     /** The settings this save changed that a running server cannot pick up. */
     const restartKeys: string[] = [];
@@ -629,10 +729,9 @@ export class DeploymentSettingsService {
       this.stored.set(key, value);
     }
 
-    // The git token is consumed through the environment (the credential helper
-    // reads `$GITHUB_TOKEN` at call time, so it never appears in argv). Putting
-    // it there is what makes a token saved here work without a restart.
-    this.syncGitTokenEnv();
+    // A token saved here is in effect at once: the git runner's credentials
+    // read `resolve('gitToken')` on every call, so nothing is published to
+    // the process environment and no restart is needed.
     return { restartRequired: restartKeys.length > 0, restartKeys };
   }
 
@@ -649,8 +748,22 @@ export class DeploymentSettingsService {
       toClear.includes(key) ? '' : (toWrite.find((w) => w.key === key)?.value ?? this.resolve(key));
   }
 
+  /**
+   * Store what the DEPLOYMENT established, internal settings included: the
+   * end of a flow that proved each value (see `SettingDef.internal`). The
+   * same validation, the same encryption, the same refusal of a setting the
+   * environment pins. Never reachable from a request body: the setup routes
+   * call {@link save}.
+   */
+  async record(entries: Record<string, string>, updatedBy: string | null): Promise<void> {
+    await this.save(entries, updatedBy, 'deployment');
+  }
+
   /** Validate a batch and return the writes (and the clears) it amounts to; throws on any problem. */
-  private plan(entries: Record<string, string>): {
+  private plan(
+    entries: Record<string, string>,
+    by: 'admin' | 'deployment' = 'admin',
+  ): {
     toWrite: { key: string; value: string; def: SettingDef }[];
     toClear: string[];
   } {
@@ -664,6 +777,12 @@ export class DeploymentSettingsService {
         problems[key] = 'Unknown setting.';
         continue;
       }
+      if (def.internal && by !== 'deployment') {
+        // Said the way an unknown setting is: what the deployment keeps for
+        // itself is not the form's to name, or to learn the names of.
+        problems[key] = 'Unknown setting.';
+        continue;
+      }
       if (this.sourceOf(key) === 'env') {
         problems[key] = `Set by the ${def.envVar} environment variable — change it there.`;
         continue;
@@ -777,18 +896,6 @@ export class DeploymentSettingsService {
     }
   }
 
-  /**
-   * Publish the resolved git token as `GITHUB_TOKEN`, the name the credential
-   * helper and every redaction path already read. Only when the environment did
-   * not supply one — otherwise this would overwrite the operator's value with
-   * a stored fallback, inverting the precedence everything else here obeys.
-   */
-  syncGitTokenEnv(): void {
-    if (this.sourceOf('gitToken') !== 'stored') return;
-    const token = this.resolve('gitToken');
-    if (token) process.env.GITHUB_TOKEN = token;
-  }
-
   /** The single sign-on values in effect, issuer normalized the way the provider uses it. */
   resolveOidcCredentials(): OidcCredentials {
     return {

package/src/modules/settings/managed-repository.ts

@@ -0,0 +1,68 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import type { IGitRunner } from '../../shared/git.contract.js';
+
+/** The folder the managed repository is kept in, under the root it is given. */
+export const MANAGED_REPOSITORY_DIR = 'knowledge-base.git';
+
+/** The branch a managed repository starts on when the deployment names none. */
+export const MANAGED_DEFAULT_BRANCH = 'main';
+
+/**
+ * The repository a deployment keeps FOR ITSELF, when its admin chose not to
+ * bring one: a bare git repository on the deployment's own disk.
+ *
+ * Nothing else about the deployment changes. Working copies are cloned from
+ * it, pushed to it and fetched from it exactly as they are from a
+ * repository on a host; it happens to be reached by a path, and to need no
+ * credential. An empty one is seeded by the startup phase like any empty
+ * remote.
+ *
+ * WHERE IT LIVES IS THE WHOLE DECISION. It is the only copy of the
+ * knowledge base that is not a working copy, so it has to be somewhere that
+ * survives the container and that nothing tidies up:
+ *
+ *  - not under the workspaces root, where the workspace service's orphan
+ *    sweep removes every folder that is not a known branch's;
+ *  - not in a new folder beside it, which every shipped compose file would
+ *    have to grow a volume for, and a deployment started from an older file
+ *    would lose on its next redeploy without a word.
+ *
+ * The composition root puts it under the BACKUPS root: a volume of its own
+ * in every compose file that has ever shipped, and one nothing sweeps.
+ */
+export class ManagedRepository {
+  /** The repository's folder: what git is given as the remote. */
+  readonly path: string;
+
+  constructor(private readonly root: string) {
+    this.path = path.join(root, MANAGED_REPOSITORY_DIR);
+  }
+
+  /** Whether the repository is there. A folder that is not a repository does not count. */
+  async exists(): Promise<boolean> {
+    return fs.access(path.join(this.path, 'HEAD')).then(
+      () => true,
+      () => false,
+    );
+  }
+
+  /**
+   * Make sure the repository exists, creating it empty when it does not.
+   * One that exists is left exactly as it is: this never re-initialises,
+   * and never touches a ref.
+   */
+  async ensure(runner: IGitRunner, initialBranch: string = MANAGED_DEFAULT_BRANCH): Promise<void> {
+    if (await this.exists()) return;
+    await fs.mkdir(this.root, { recursive: true });
+    // A folder that is there without being a repository is somebody's, and
+    // initialising over it would mix a repository into their files.
+    const entries = await fs.readdir(this.path).catch(() => null);
+    if (entries && entries.length > 0) {
+      throw new Error(
+        `${this.path} exists and is not a git repository. Move it away, or empty it, before choosing a managed repository.`,
+      );
+    }
+    await runner.run(this.root, ['init', '--bare', `--initial-branch=${initialBranch}`, this.path]);
+  }
+}

package/src/modules/settings/repository-source.ts

@@ -0,0 +1,197 @@
+import { DEFAULT_GIT_USERNAME, type GitCredentials } from '../../shared/git.contract.js';
+import type { ManagedRepository } from './managed-repository.js';
+
+/**
+ * The ways a deployment can be given the repository its knowledge base
+ * lives in, in the order the setup screen offers them.
+ *
+ *  - `managed`: the deployment keeps the repository itself.
+ *  - `github-app`: a repository on GitHub, reached through a GitHub App the
+ *    admin installed on it.
+ *  - `token`: any git host, by its address and an access token.
+ */
+export const GIT_MODES = ['managed', 'github-app', 'token'] as const;
+export type GitMode = (typeof GIT_MODES)[number];
+
+export function isGitMode(value: unknown): value is GitMode {
+  return typeof value === 'string' && (GIT_MODES as readonly string[]).includes(value);
+}
+
+/** Reads one setting as it is in effect, or as it would be after a save. */
+export type SettingReader = (key: string) => string;
+
+/**
+ * A repository reached through a GitHub App: which one, and the short-lived
+ * token the app is given for it. Implemented by the GitHub App module; the
+ * source only asks.
+ */
+export interface GitHubAppRepository {
+  /** `https://github.com/<owner>/<name>.git`, or empty while none is chosen. */
+  url(read: SettingReader): string;
+  /** The installation token in hand, or null when there is none to give. */
+  token(): string | null;
+  /**
+   * Make sure {@link token} answers with one that is still good. `asked`
+   * is for a caller waiting on the answer itself, who is told a failure.
+   */
+  prepare(opts?: { asked?: boolean }): Promise<void>;
+  /** Whether the deployment may be pointed at this repository (`owner/name`). */
+  permits(repository: string, read: SettingReader): boolean;
+  /** Whether the app, its installation and a repository are all there. */
+  answered(read: SettingReader): boolean;
+}
+
+export interface RepositorySourceOptions {
+  /** The deployment's settings, as they are in effect. */
+  read: SettingReader;
+  /**
+   * What the environment gives the token mode beyond the settings
+   * catalogue: the legacy spellings of the token variable, and the username
+   * a deployment configured there.
+   */
+  fallback?: { username?: string; token?: string };
+  managed: ManagedRepository;
+  githubApp?: GitHubAppRepository;
+}
+
+/**
+ * WHERE THE KNOWLEDGE BASE'S REPOSITORY IS, AND WHAT GIT PRESENTS TO IT —
+ * the one place that knows there is more than one answer.
+ *
+ * Everything that runs git asks for two things: an address to clone from,
+ * and a credential. Both used to be read straight from two settings, which
+ * is one way of having a repository. With three ways, each reader deciding
+ * for itself would be three places to disagree, so they all ask here, and
+ * what they get back is the same two things as before.
+ *
+ * TWO MODES, AND THEY ARE NOT ALWAYS THE SAME ONE.
+ *
+ *  - The mode IN EFFECT is the one this process is running on. Its working
+ *    copies are clones of that mode's repository, and every address and
+ *    credential this class hands out is that mode's. It is taken when the
+ *    source is built and changes only in {@link takeEffect}.
+ *  - The mode CHOSEN is what the settings say. A save changes it at once.
+ *
+ * They differ between the save that moves a serving deployment and the
+ * restart that save owes. Reading the mode live would move the process the
+ * moment the setting was stored: the new mode's credential presented to the
+ * old mode's repository, which every working copy still points at, and new
+ * branches cloned from a repository the startup phase has not prepared. So
+ * the process stays where it is until it is started again, which is what
+ * the setup screen tells the admin.
+ *
+ * WITHIN a mode the values are read live, as they always were: a rotated
+ * token is the one the next push carries.
+ *
+ * THE MODE IS INFERRED WHEN NOBODY CHOSE ONE. A deployment configured before
+ * there was a choice has an address and a token and no mode; it is in
+ * `token` mode, because that is what it has. One with nothing configured has
+ * no mode, and the setup screen offers the first.
+ */
+export class RepositorySource {
+  /** What git authenticates with, read on every call. */
+  readonly credentials: GitCredentials;
+  private inEffect: GitMode | null;
+
+  constructor(private readonly opts: RepositorySourceOptions) {
+    this.inEffect = this.chosen();
+    this.credentials = {
+      username: () => this.username(),
+      token: () => this.token(),
+      prepare: () => this.prepare(),
+    };
+  }
+
+  /**
+   * Without a reader, the mode IN EFFECT. With one, the mode a save of that
+   * reader's values would CHOOSE: the answer validation wants, about values
+   * that are not stored yet. Null: nothing is configured.
+   */
+  mode(read?: SettingReader): GitMode | null {
+    return read ? this.chosen(read) : this.inEffect;
+  }
+
+  /** The mode the settings say, which a restart would put in effect. */
+  chosen(read: SettingReader = this.opts.read): GitMode | null {
+    const named = read('gitMode').trim();
+    if (isGitMode(named)) return named;
+    return read('kbRepoUrl') || this.tokenIn(read) ? 'token' : null;
+  }
+
+  /**
+   * Put the mode chosen in effect. Called where nothing is using the mode
+   * that was: by a save made before the deployment was serving, which the
+   * startup phase follows. A process that is serving is never moved; it is
+   * restarted, and built on the mode chosen.
+   */
+  takeEffect(): void {
+    this.inEffect = this.chosen();
+  }
+
+  /** The address git clones from and pushes to. Empty while there is none. Asked as {@link mode} is. */
+  url(read?: SettingReader): string {
+    const from = read ?? this.opts.read;
+    switch (this.mode(read)) {
+      case 'managed':
+        return this.opts.managed.path;
+      case 'github-app':
+        return this.opts.githubApp?.url(from) ?? '';
+      case 'token':
+        return from('kbRepoUrl');
+      default:
+        return '';
+    }
+  }
+
+  /**
+   * Whether the repository half of setup is answered: there is somewhere to
+   * clone from and, where one is needed, something to present. Asked as
+   * {@link mode} is.
+   */
+  answered(read?: SettingReader): boolean {
+    const from = read ?? this.opts.read;
+    switch (this.mode(read)) {
+      case 'managed':
+        return true;
+      case 'github-app':
+        return this.opts.githubApp?.answered(from) ?? false;
+      case 'token':
+        return Boolean(from('kbRepoUrl') && this.tokenIn(from));
+      default:
+        return false;
+    }
+  }
+
+  private username(): string {
+    // Only a host that was given a token is given a name to go with it.
+    if (this.inEffect !== 'token') return DEFAULT_GIT_USERNAME;
+    return this.opts.read('gitUsername') || this.opts.fallback?.username || DEFAULT_GIT_USERNAME;
+  }
+
+  /**
+   * One mode's credential is never presented to another mode's repository:
+   * a deployment that moved to a managed repository still has its old
+   * token stored, and a path on its own disk has no use for it. The mode is
+   * the one IN EFFECT, because that is the repository git is talking to.
+   */
+  private token(): string | null {
+    switch (this.inEffect) {
+      case 'token':
+        return this.tokenIn(this.opts.read) || null;
+      case 'github-app':
+        return this.opts.githubApp?.token() ?? null;
+      default:
+        return null;
+    }
+  }
+
+  private async prepare(): Promise<void> {
+    if (this.inEffect === 'github-app') await this.opts.githubApp?.prepare();
+  }
+
+  private tokenIn(read: SettingReader): string {
+    // The fallback is the environment's, so it is in effect whatever a save brings.
+    return read('gitToken') || this.opts.fallback?.token || '';
+  }
+}
+