@bevel-software/platform-core-backend 0.22.0 → 0.24.0

package/src/modules/database/connection.ts

@@ -2,6 +2,7 @@ import { sql } from 'drizzle-orm';
 import { drizzle } from 'drizzle-orm/node-postgres';
 import pg from 'pg';
 import * as schema from './schema.js';
+import { PII_CIPHERTEXT_PREFIX, PiiParam, derivePiiKeys, type PiiKeys } from '../../shared/column-crypto.js';
 
 /**
  * One database, one schema per knowledge base.
@@ -74,6 +75,83 @@ export interface DbOptions {
   max?: number;
   /** How long an idle connection is kept before the pool closes it. Default: `pg`'s. */
   idleTimeoutMillis?: number;
+  /**
+   * The key this handle seals personal data with: the `SECRETS_ENC_KEY` of
+   * the knowledge base whose rows it holds (its own derived one, for a
+   * tenant). Without it the handle reads sealed values as their blobs and
+   * refuses to write one — right for a handle that only touches tables with
+   * no personal data, and a loud failure for one that was meant to have it.
+   */
+  piiKey?: string;
+}
+
+type QueryFn = (this: pg.Client, config?: unknown, values?: unknown, callback?: unknown) => unknown;
+type ResultCallback = (err: unknown, result: unknown) => void;
+
+/**
+ * The `pg` client of a handle that holds `keys`: the one place personal data
+ * changes form. Going in, every value a personal-data column marked (see
+ * `column-crypto.ts`) is replaced by its ciphertext or its blind index.
+ * Coming out, every sealed value of a result is opened — by its prefix, not
+ * by its column, so raw SQL through the handle reads plaintext too; a
+ * statement that needs what is STORED asks for it as bytes
+ * (`convert_to(col, 'UTF8')`).
+ *
+ * At the client, because that is what the handle owns: the pool builds every
+ * connection from this class, a transaction's included, so there is no path
+ * from this handle to the database that goes round it. Results that are
+ * streamed to a `Submittable` (a cursor) are not opened; nothing here uses one.
+ */
+function keyedClient(keys: PiiKeys | null): typeof pg.Client {
+  const stored = (value: unknown): unknown => {
+    if (!(value instanceof PiiParam)) return value;
+    return keys ? value.stored(keys) : value.toPostgres();
+  };
+  const openCell = (value: unknown): unknown =>
+    typeof value === 'string' && value.startsWith(PII_CIPHERTEXT_PREFIX) ? keys!.read(value) : value;
+  const openOne = (result: unknown): void => {
+    const rows = (result as { rows?: unknown } | null)?.rows;
+    if (!Array.isArray(rows)) return;
+    for (const row of rows) {
+      if (Array.isArray(row)) {
+        for (let i = 0; i < row.length; i++) row[i] = openCell(row[i]);
+      } else if (row && typeof row === 'object') {
+        const record = row as Record<string, unknown>;
+        for (const column of Object.keys(record)) record[column] = openCell(record[column]);
+      }
+    }
+  };
+  const open = <T,>(result: T): T => {
+    if (!keys) return result;
+    // Several statements in one query string come back as several results.
+    if (Array.isArray(result)) result.forEach(openOne);
+    else openOne(result);
+    return result;
+  };
+
+  const base = pg.Client.prototype.query as unknown as QueryFn;
+  class KeyedClient extends pg.Client {}
+  const query: QueryFn = function (config, values, callback) {
+    if (typeof values === 'function') {
+      callback = values;
+      values = undefined;
+    }
+    if (Array.isArray(values)) {
+      values = values.map(stored);
+    } else if (config && typeof config === 'object' && Array.isArray((config as { values?: unknown }).values)) {
+      const withValues = config as { values: unknown[]; submit?: unknown };
+      if (typeof withValues.submit === 'function') withValues.values = withValues.values.map(stored);
+      else config = { ...withValues, values: withValues.values.map(stored) };
+    }
+    if (typeof callback === 'function') {
+      const done = callback as ResultCallback;
+      return base.call(this, config, values, (err: unknown, result: unknown) => done(err, err ? result : open(result)));
+    }
+    const pending = base.call(this, config, values);
+    return pending instanceof Promise ? pending.then(open) : pending;
+  };
+  Object.defineProperty(KeyedClient.prototype, 'query', { value: query, writable: true, configurable: true });
+  return KeyedClient;
 }
 
 /**
@@ -96,7 +174,11 @@ const CONNECT_TIMEOUT_MS = 30_000;
  */
 export function createDb(databaseUrl: string, opts: DbOptions = {}) {
   const dbSchema = assertSchemaName(opts.schema ?? DEFAULT_DB_SCHEMA);
+  // Only an ABSENT key means "this handle holds none". A blank or malformed
+  // one was meant to be a key, and is refused here rather than read as none.
+  const keys = opts.piiKey === undefined ? null : derivePiiKeys(opts.piiKey);
   const pool = new pg.Pool({
+    Client: keyedClient(keys),
     connectionString: databaseUrl,
     connectionTimeoutMillis: CONNECT_TIMEOUT_MS,
     max: opts.max,
@@ -109,6 +191,7 @@ export function createDb(databaseUrl: string, opts: DbOptions = {}) {
   });
   const db = drizzle(pool, { schema });
   schemas.set(db, dbSchema);
+  if (keys) piiKeys.set(db, { keys, of: opts.piiKey! });
   return db;
 }
 
@@ -119,7 +202,22 @@ export function dbSchemaOf(db: Pick<Database, '$client'>): string {
   return schemas.get(db as Database) ?? DEFAULT_DB_SCHEMA;
 }
 
+/**
+ * The keys a handle seals personal data with — for the few callers that
+ * handle the stored form themselves (the backfill at start; an overlay's own
+ * SQL). Everything else never sees a key: the column types and the handle's
+ * connection do the work. Throws for a handle built without one.
+ */
+export function piiKeysOf(db: Pick<Database, '$client'>): PiiKeys {
+  const held = piiKeys.get(db as Database);
+  if (!held) {
+    throw new Error('This database handle holds no personal-data key: build it with createDb/getDb and its piiKey.');
+  }
+  return held.keys;
+}
+
 const schemas = new WeakMap<Database, string>();
+const piiKeys = new WeakMap<Database, { keys: PiiKeys; of: string }>();
 const cache = new Map<string, Database>();
 
 const cacheKey = (databaseUrl: string, dbSchema: string) => `${dbSchema}\u0000${databaseUrl}`;
@@ -136,10 +234,29 @@ export function getDb(databaseUrl: string, opts: DbOptions = {}): Database {
   if (!db) {
     db = createDb(databaseUrl, opts);
     cache.set(key, db);
+  } else if (opts.piiKey !== undefined && !holdsKey(db, opts.piiKey)) {
+    // One schema is one knowledge base and one key. Handing back the cached
+    // pool would seal this caller's rows under the other caller's key.
+    throw new Error(`The database handle for schema "${dbSchema}" is already open with a different personal-data key.`);
   }
   return db;
 }
 
+/**
+ * Whether `db` was opened with the key `piiKey` spells. The KEY, not its
+ * spelling: `SECRETS_ENC_KEY` may be written as hex or as base64, and one
+ * process can be handed the same 32 bytes both ways — refusing the second as
+ * "a different key" would refuse a caller that holds the right one. Compared
+ * through what the two derive, so no key material is held here to compare.
+ */
+function holdsKey(db: Database, piiKey: string): boolean {
+  const held = piiKeys.get(db);
+  if (!held) return false;
+  if (held.of === piiKey) return true;
+  const probe = 'same-key?';
+  return held.keys.index(probe) === derivePiiKeys(piiKey).index(probe);
+}
+
 /**
  * End a handle's pool and forget it, so a later {@link getDb} for the same
  * schema opens a fresh one rather than handing out a pool that was ended.

package/src/modules/database/core-schema.ts

@@ -1,23 +1,40 @@
 /**
  * CORE platform tables — the git-backed workspace/workflow, auth, access,
- * change requests, locks, the pending-commits queue, the ontology-session
- * touched-set, connection keys, MCP OAuth, and the Secrets Vault. A core-only
- * deployment migrates and runs on exactly these tables.
+ * change requests, locks, the pending-commits queue, connection keys, MCP
+ * OAuth, and the Secrets Vault. A core-only deployment migrates and runs on
+ * exactly these tables.
  *
  * Enterprise-only tables (chat, routines, watchlist, connectors, LLM config,
  * SharePoint/Atlassian links, feedback, upload, kb-revalidation) live in
  * `enterprise-schema.ts`, which imports the FK targets (`users`,
  * `externalApiKeys`) from here. `schema.ts` re-exports both, so existing
  * imports keep working unchanged.
+ *
+ * PII columns (emails, display names, change-request/comment text) are
+ * `encryptedText` — AES-256-GCM ciphertext in the database, transparently
+ * decrypted on read (see `shared/column-crypto.ts`). Because the ciphertext is
+ * randomized, equality lookups and unique constraints on those columns go
+ * through their deterministic `*_bidx` companions (HMAC-SHA256 blind indexes,
+ * `blindIndexText`) — never `eq()` an encrypted column directly. A `*_bidx`
+ * column is written and compared WITH THE ADDRESS ITSELF; the database handle
+ * the statement runs on turns it into the index, under that handle's key.
  */
 import { sql } from 'drizzle-orm';
 import { boolean, check, index, integer, jsonb, pgTable, primaryKey, text, timestamp, uniqueIndex, uuid } from 'drizzle-orm/pg-core';
+import { blindIndexText, encryptedText } from '../../shared/column-crypto.js';
 
 export const users = pgTable('users', {
   id: uuid('id').defaultRandom().primaryKey(),
-  email: text('email').notNull().unique(),
-  name: text('name').notNull(),
-  avatarUrl: text('avatar_url'),
+  email: encryptedText('email').notNull(),
+  /**
+   * Blind index of the lowercased email — carries the uniqueness constraint
+   * and every lookup-by-email (login, upsert conflict target), which the
+   * randomized `email` ciphertext cannot. Write the email into it on every
+   * insert, and compare it with the email.
+   */
+  emailBidx: blindIndexText('email_bidx').notNull(),
+  name: encryptedText('name').notNull(),
+  avatarUrl: encryptedText('avatar_url'),
   /**
    * scrypt hash for password login (see auth/password-hash.ts). NULL for
    * accounts that only ever signed in via SSO — password login refuses them
@@ -37,9 +54,24 @@ export const users = pgTable('users', {
    * backfill would need, and being shown the setup once costs a click.
    */
   onboardingDone: boolean('onboarding_done').default(false).notNull(),
+  /**
+   * When an admin switched this account off; NULL while it is on. A
+   * deactivated account keeps its row, its history and its place in roles
+   * and groups, but nothing it holds is honoured: it cannot sign in, and its
+   * session, connection keys, agent tokens and internal tokens are refused
+   * the next time they are presented (see `AuthService.isActive`). Turning it
+   * back on restores all of them as they were.
+   *
+   * The off switch a host that sells seats needs — a seat is an account that
+   * is on — and the one an admin uses for someone who left without erasing
+   * what they did.
+   */
+  deactivatedAt: timestamp('deactivated_at'),
   createdAt: timestamp('created_at').defaultNow().notNull(),
   updatedAt: timestamp('updated_at').defaultNow().notNull(),
-});
+}, (t) => ({
+  emailBidxUnq: uniqueIndex('users_email_bidx_unq').on(t.emailBidx),
+}));
 
 /**
  * Per-file owner approvals on a PR. A PR is mergeable-in-Bevel when every
@@ -56,15 +88,18 @@ export const prFileApprovals = pgTable('pr_file_approvals', {
   id: uuid('id').defaultRandom().primaryKey(),
   prNumber: integer('pr_number').notNull(),
   path: text('path').notNull(),
-  approverEmail: text('approver_email').notNull(),  // lowercased at insert
-  approverName: text('approver_name').notNull(),
+  approverEmail: encryptedText('approver_email').notNull(),  // lowercased at insert
+  /** Blind index of `approver_email` — the uniqueness key and eq() lookup. */
+  approverEmailBidx: blindIndexText('approver_email_bidx').notNull(),
+  approverName: encryptedText('approver_name').notNull(),
   headSha: text('head_sha').notNull(),
   approvedAt: timestamp('approved_at').defaultNow().notNull(),
 }, (t) => ({
   // Idempotency: approving the same path on the same SHA twice must be a no-op,
-  // not a duplicate row. Unique on the (PR, path, approver, headSha) tuple.
-  unq: uniqueIndex('pr_file_approvals_unq')
-    .on(t.prNumber, t.path, t.approverEmail, t.headSha),
+  // not a duplicate row. Unique on the (PR, path, approver, headSha) tuple —
+  // via the blind index, because the encrypted email is randomized.
+  unq: uniqueIndex('pr_file_approvals_bidx_unq')
+    .on(t.prNumber, t.path, t.approverEmailBidx, t.headSha),
   byPr: index('pr_file_approvals_by_pr').on(t.prNumber),
 }));
 
@@ -78,12 +113,15 @@ export const prFileApprovals = pgTable('pr_file_approvals', {
 export const prMergeLog = pgTable('pr_merge_log', {
   id: uuid('id').defaultRandom().primaryKey(),
   prNumber: integer('pr_number').notNull(),
-  triggeredByEmail: text('triggered_by_email').notNull(),
-  triggeredByName: text('triggered_by_name').notNull(),
+  triggeredByEmail: encryptedText('triggered_by_email').notNull(),
+  /** Blind index of `triggered_by_email` — GDPR-erasure lookup. */
+  triggeredByEmailBidx: blindIndexText('triggered_by_email_bidx').notNull(),
+  triggeredByName: encryptedText('triggered_by_name').notNull(),
   headShaAtMerge: text('head_sha_at_merge').notNull(),
   mergeMethod: text('merge_method').notNull(),
   succeeded: boolean('succeeded').notNull(),
-  error: text('error'),
+  // Encrypted: merge failure output can quote author identities and CR text.
+  error: encryptedText('error'),
   startedAt: timestamp('started_at').defaultNow().notNull(),
   completedAt: timestamp('completed_at'),
 }, (t) => ({
@@ -107,12 +145,15 @@ export const prMergeLog = pgTable('pr_merge_log', {
 export const prComments = pgTable('pr_comments', {
   id: uuid('id').defaultRandom().primaryKey(),
   prNumber: integer('pr_number').notNull(),
-  authorEmail: text('author_email').notNull(),
-  authorName: text('author_name').notNull(),
+  authorEmail: encryptedText('author_email').notNull(),
+  /** Blind index of `author_email` — GDPR-erasure lookup. */
+  authorEmailBidx: blindIndexText('author_email_bidx').notNull(),
+  authorName: encryptedText('author_name').notNull(),
   path: text('path'),
   line: integer('line'),
   headSha: text('head_sha').notNull(),
-  body: text('body').notNull(),
+  // Encrypted: review conversation exists only in this DB (never on GitHub).
+  body: encryptedText('body').notNull(),
   parentId: uuid('parent_id'),
   createdAt: timestamp('created_at').defaultNow().notNull(),
   updatedAt: timestamp('updated_at'),
@@ -141,25 +182,53 @@ export const changeRequests = pgTable('change_requests', {
   number: integer('number').generatedAlwaysAsIdentity(),
   sourceBranch: text('source_branch').notNull(),
   targetBranch: text('target_branch').notNull(),
-  title: text('title').notNull(),
-  body: text('body').notNull().default(''),
-  authorEmail: text('author_email').notNull(), // lowercased at insert
-  authorName: text('author_name').notNull(),
+  // Encrypted: CR title/body exist only in this DB (the remote sees branches).
+  // The DB-level default '' bypasses toDriver, which is fine: the empty string
+  // is deliberately stored unencrypted (see encryptPii), so a defaulted row
+  // and an app-written empty body hold the identical representation.
+  title: encryptedText('title').notNull(),
+  body: encryptedText('body').notNull().default(''),
+  authorEmail: encryptedText('author_email').notNull(), // lowercased at insert
+  /** Blind index of `author_email` — GDPR-erasure lookup. */
+  authorEmailBidx: blindIndexText('author_email_bidx').notNull(),
+  authorName: encryptedText('author_name').notNull(),
   state: text('state').notNull().default('open'), // 'open' | 'merged' | 'closed'
   mergedSha: text('merged_sha'),
   // The last apply attempt that did not land (null when none, or once a gate
   // input it depended on changed). Persisted rather than only pushed to the
   // clicker so every viewer of the still-open request — its author first —
   // sees the refusal.
-  applyFailureReason: text('apply_failure_reason'),
+  // Encrypted: the reason can quote identities the way the merge log's error
+  // does, and the name is a person's.
+  applyFailureReason: encryptedText('apply_failure_reason'),
   applyFailureConflicts: boolean('apply_failure_conflicts'),
   applyFailedAt: timestamp('apply_failed_at'),
-  applyFailedByName: text('apply_failed_by_name'),
+  applyFailedByName: encryptedText('apply_failed_by_name'),
+  /**
+   * Blind index of the email of the person named above — what account erasure
+   * finds their name by. Written and cleared with the name, so a name never
+   * stands without it: a refusal recorded before the encryption release kept
+   * no address, and its name is taken off at the first start on this release
+   * (`clearUnindexedRefusalNames` in `migrate.ts`).
+   */
+  applyFailedByEmailBidx: blindIndexText('apply_failed_by_email_bidx'),
   /** What refused the last apply: 'gate' (approvals), 'conflicts' (git), 'error' (anything else). */
   applyFailureKind: text('apply_failure_kind'),
   createdAt: timestamp('created_at').defaultNow().notNull(),
   updatedAt: timestamp('updated_at'),
   closedAt: timestamp('closed_at'),
+  /**
+   * Why a closed request was closed, when something other than a person
+   * closed it. Today the one value is 'repository-replaced': the admin
+   * pointed the deployment at a different knowledge-base repository and chose
+   * to close the requests rather than keep them. Null for every request
+   * closed the ordinary way — by its author, or by the deleted-branch sweep —
+   * so the column reads as "nobody recorded a reason", not as a default.
+   *
+   * The row is never deleted: a replaced repository loses the branches, not
+   * the record of what people asked for.
+   */
+  closedReason: text('closed_reason'),
 }, (t) => ({
   numberUnq: uniqueIndex('change_requests_number_unq').on(t.number),
   // Enforce the "A→B blocks A→B while open" uniqueness rule at the DB level —
@@ -245,7 +314,7 @@ export const fileLocks = pgTable('file_locks', {
   branch: text('branch').notNull(),
   path: text('path').notNull(),
   holderUserId: uuid('holder_user_id').notNull().references(() => users.id),
-  holderName: text('holder_name').notNull(),
+  holderName: encryptedText('holder_name').notNull(),
   // How the lock was acquired. 'edit' (the default) is a normal write hold —
   // the holder is editing and the release publishes their bytes.
   // 'coordination' is a pure-mutex hold (see IWorkflowService.acquireLock)
@@ -306,8 +375,10 @@ export const pendingCommits = pgTable('pending_commits', {
   // Author attribution captured at enqueue. The denormalised name/email pair
   // lands on the eventual `git commit --author=` so the commit history shows
   // the human who triggered the save, not the worker.
-  authorEmail: text('author_email').notNull(),  // lowercased at insert
-  authorName: text('author_name').notNull(),
+  authorEmail: encryptedText('author_email').notNull(),  // lowercased at insert
+  /** Blind index of `author_email` — GDPR-erasure lookup. */
+  authorEmailBidx: blindIndexText('author_email_bidx').notNull(),
+  authorName: encryptedText('author_name').notNull(),
   queuedAt: timestamp('queued_at').defaultNow().notNull(),
   // `running` is set while the worker is mid-commit so a second worker
   // (e.g. after a restart with another instance still draining) doesn't
@@ -320,7 +391,8 @@ export const pendingCommits = pgTable('pending_commits', {
   // `needs_attention` + feedback notice instead of looping forever.
   recoveryAgentRuns: integer('recovery_agent_runs').notNull().default(0),
   lastAttemptedAt: timestamp('last_attempted_at'),
-  lastError: text('last_error'),
+  // Encrypted: git/agent failure output can quote author identities and paths.
+  lastError: encryptedText('last_error'),
 }, (t) => ({
   // Primary read pattern: drain in queued order, scoped to a workspace.
   byWorkspaceQueued: index('pending_commits_by_workspace_queued').on(t.workspaceId, t.queuedAt),
@@ -382,30 +454,6 @@ export const secrets = pgTable('secrets', {
   sharedKeyUnq: uniqueIndex('secrets_shared_key_unq').on(t.key).where(sql`${t.userId} is null`),
 }));
 
-/**
- * Ontology-session boundary — the per-session log of which named ontologies a
- * run has TOUCHED (one row per `(session_id, ontology)`). Each agent operation
- * that resolves to a named ontology records a row here (idempotent on the PK).
- *
- * The rule the gate enforces from this set:
- *   - READS are always allowed; they just record the ontology they touched.
- *   - WRITES are allowed only while the session's touched set is a SINGLE named
- *     ontology equal to the write target. Once a session has touched two or
- *     more ontologies (e.g. by reading across them), every write is blocked —
- *     "read across ontologies → you can't write anymore at all."
- *
- * Postgres is the durable source of truth so the boundary holds across a
- * backend restart (an in-memory set would reset and silently un-poison a run).
- * A write-through in-process cache fronts this table for the chatty read path;
- * a cache miss falls back here.
- *
- *   - PK is `(session_id, ontology)` — a set, idempotent on repeat touches.
- *   - `ontology` is the resolved ontology id (e.g. `KnowledgeBase/Product`).
- *     Neutral touches don't record, so they never write a row.
- *   - `touched_at` drives the abandoned-run `sweepOlderThan` backstop; the
- *     primary reclamation is an explicit delete of all of a session's rows at
- *     run end.
- */
 /**
  * OAuth clients dynamically registered by MCP clients (RFC 7591 DCR). When an
  * MCP client (claude.ai, Claude Code) connects to `/api/mcp` without a
@@ -575,6 +623,13 @@ export const agentEvents = pgTable('agent_events', {
   ),
 }));
 
+/**
+ * UNUSED IN THIS RELEASE. It backed the per-session boundary that a deployment
+ * now owns for itself, behind the agent read/write hooks; nothing in this
+ * package reads or writes it any more. The definition stays for ONE release so
+ * that no deployment runs a version in which neither side has the table, and
+ * the NEXT release drops it (see `TODOS.md`). Do not build on it.
+ */
 export const sessionOntologyTouches = pgTable('session_ontology_touches', {
   sessionId: text('session_id').notNull(),
   ontology: text('ontology').notNull(),
@@ -710,13 +765,15 @@ export const githubFacadeCodes = pgTable('github_facade_codes', {
  */
 export const pluginJoinRequests = pgTable('plugin_join_requests', {
   id: uuid('id').defaultRandom().primaryKey(),
-  requesterEmail: text('requester_email').notNull(), // lowercased at insert
+  requesterEmail: encryptedText('requester_email').notNull(), // lowercased at insert
+  /** Blind index of `requester_email` — the uniqueness key, the by-requester read and the erasure delete. */
+  requesterEmailBidx: blindIndexText('requester_email_bidx').notNull(),
   /** Denormalised for the commit/change-request authorship, like `file_locks.holder_name`. */
-  requesterName: text('requester_name').notNull(),
+  requesterName: encryptedText('requester_name').notNull(),
   pluginKey: text('plugin_key').notNull(),
   status: text('status').notNull().default('pending'),
-  /** What the git work said when it refused — shown to the requester verbatim. */
-  failureReason: text('failure_reason'),
+  /** What the git work said when it refused — shown to the requester verbatim. Encrypted: git quotes identities. */
+  failureReason: encryptedText('failure_reason'),
   changeRequestNumber: integer('change_request_number'),
   /**
    * When a process took this row's git work, and the whole of the mutual
@@ -762,8 +819,9 @@ export const pluginJoinRequests = pgTable('plugin_join_requests', {
 }, (t) => ({
   // One request per person per plugin — the DB's rule, not a caller's. Also
   // the index the plugin listing's by-requester read is served from.
-  requesterPluginUnq: uniqueIndex('plugin_join_requests_requester_plugin_unq')
-    .on(t.requesterEmail, t.pluginKey),
+  // Via the blind index, because the encrypted email is randomized.
+  requesterPluginUnq: uniqueIndex('plugin_join_requests_requester_bidx_plugin_unq')
+    .on(t.requesterEmailBidx, t.pluginKey),
   // The boot sweep: every row still `pending`, without a full scan.
   byStatus: index('plugin_join_requests_by_status').on(t.status),
   statusCheck: check(