@proteinjs/db 1.35.1 → 1.36.0

package/src/encryption/EncryptionRecordHooks.ts

@@ -0,0 +1,167 @@
+import type { Table } from '../Table';
+import type { SerializedRecord } from '../Record';
+import { EncryptedColumnConfigError, getDbEncryptionConfig } from './DbEncryptionConfig';
+import { EncryptedColumns } from './EncryptedColumns';
+import { EncryptionEnvelope } from './EncryptionEnvelope';
+import { DataKeyMaterial, DataKeyStore } from './DataKeyStore';
+import { SearchTokenizer } from './SearchTokenizer';
+
+/** Who a row being written belongs to — the data key its values encrypt under. */
+export type EncryptionWriteContext = { keyOwner: string };
+
+/**
+ * The transparent encrypt/decrypt seam, invoked by `RecordSerializer` on every write's
+ * serialize and every read's deserialize — the same per-column hook layer every caller
+ * already passes through, so services, watchers, migrations, and queued client
+ * transactions get encryption without knowing it exists.
+ *
+ * - Serialize: after the columns' own `serialize` steps, values of encrypted columns are
+ *   replaced with self-describing envelopes (`EncryptionEnvelope`), and the searchable /
+ *   sort companions are written beside them in the same statement.
+ * - Deserialize: before the columns' own `deserialize` steps, envelope values are decrypted
+ *   (the envelope names its key — owner + version — so no context is needed) and
+ *   framework-derived companion columns are dropped. A non-envelope value in an encrypted
+ *   column passes through unchanged — the online-adoption transition: rows written before
+ *   the encrypt backfill stay readable while the walker converges them.
+ *
+ * Server-only: this seam executes where the database driver runs. Browser code proxies
+ * through `DbService` and receives decrypted values over the authenticated service.
+ */
+export class EncryptionRecordHooks {
+  private encryptedColumns = new EncryptedColumns();
+  private envelope = new EncryptionEnvelope();
+  private tokenizer = new SearchTokenizer();
+
+  /** Encrypt encrypted-column values in `serialized` (column-name-keyed) and add companions. */
+  async onSerialize(table: Table<any>, serialized: SerializedRecord, context?: EncryptionWriteContext): Promise<void> {
+    this.encryptedColumns.ensureSchema(table);
+    const touchedProps = this.encryptedColumns
+      .encryptedProps(table)
+      .filter((prop) => ((table.columns as any)[prop] as { name: string }).name in serialized);
+    if (touchedProps.length === 0) {
+      return;
+    }
+
+    this.assertServerSide();
+    let writeKey: DataKeyMaterial | undefined;
+    for (const prop of touchedProps) {
+      const column = (table.columns as any)[prop];
+      const config = this.encryptedColumns.configFor(table, prop)!;
+      const value = serialized[column.name];
+      if (value === null) {
+        // Null stays null (IS NULL queries keep working); companions null out with it.
+        if (config.searchable === 'equality') {
+          serialized[`${column.name}_enc_eq`] = null;
+        }
+        if (config.sortKey) {
+          serialized[`${column.name}_enc_srt`] = null;
+        }
+        continue;
+      }
+
+      if (typeof value !== 'string') {
+        throw new EncryptedColumnConfigError(
+          `(${table.name}.${prop}) encrypted columns must serialize to a string; ` +
+            `got ${typeof value}. Declare 'encrypted' on a StringColumn-family column.`
+        );
+      }
+
+      if (!writeKey) {
+        if (!context?.keyOwner) {
+          throw new EncryptedColumnConfigError(
+            `(${table.name}) A write touches encrypted column '${prop}' but no key owner was ` +
+              `resolved. Writes to encrypted columns must resolve the row's scope owner ` +
+              `(the row's 'scope' value, or DbEncryptionConfig.resolveKeyOwner).`
+          );
+        }
+        writeKey = await new DataKeyStore().getWriteKey(context.keyOwner);
+      }
+
+      serialized[column.name] = this.envelope.encrypt(value, writeKey);
+      if (config.searchable === 'equality') {
+        serialized[`${column.name}_enc_eq`] = this.tokenizer.equalityFingerprint(value, writeKey.indexKey);
+      }
+      if (config.sortKey) {
+        serialized[`${column.name}_enc_srt`] = this.tokenizer.sortPrefix(value, config.sortKey.revealPrefix);
+      }
+    }
+  }
+
+  /**
+   * A copy of `serializedRecord` with framework companion columns dropped and every
+   * envelope value decrypted (any column — self-describing ciphertext also covers the
+   * decrypt-out transition, where the config no longer marks the column).
+   */
+  async onDeserialize(table: Table<any>, serializedRecord: SerializedRecord): Promise<SerializedRecord> {
+    this.encryptedColumns.ensureSchema(table);
+    const prepared: SerializedRecord = {};
+    const internalColumnNames = this.internalColumnNames(table);
+    let decrypted = false;
+    for (const columnName of Object.keys(serializedRecord)) {
+      if (internalColumnNames.has(columnName)) {
+        continue;
+      }
+
+      const value = serializedRecord[columnName];
+      const parsed = this.envelope.parse(value);
+      if (!parsed) {
+        prepared[columnName] = value;
+        continue;
+      }
+
+      this.assertServerSide();
+      const key = await new DataKeyStore().getKeyByVersion(parsed.owner, parsed.version);
+      prepared[columnName] = this.envelope.decrypt(value as string, key);
+      decrypted = true;
+    }
+
+    return decrypted || internalColumnNames.size > 0 ? prepared : serializedRecord;
+  }
+
+  /**
+   * The key owner for a row being written — its permission-source scope owner:
+   * `DbEncryptionConfig.resolveKeyOwner` when the deployment supplies one (richer sharing
+   * models map scope→owner there), else the row's `scope` column value (the framework's
+   * scope columns hold the owning user's id).
+   */
+  async resolveKeyOwnerForWrite(table: Table<any>, record: any): Promise<string> {
+    const config = getDbEncryptionConfig();
+    if (config.resolveKeyOwner) {
+      const owner = await config.resolveKeyOwner({ table, record });
+      if (owner) {
+        return owner;
+      }
+    }
+
+    if (typeof record.scope === 'string' && record.scope.length > 0) {
+      return record.scope;
+    }
+
+    throw new EncryptedColumnConfigError(
+      `(${table.name}) Cannot resolve the key owner for a write to an encrypted column: the row ` +
+        `carries no 'scope' value and DbEncryptionConfig.resolveKeyOwner resolved nothing. ` +
+        `Scoped tables get this automatically; other tables must supply resolveKeyOwner.`
+    );
+  }
+
+  private internalColumnNames(table: Table<any>): Set<string> {
+    const names = new Set<string>();
+    for (const prop of Object.keys(table.columns)) {
+      const column = (table.columns as any)[prop];
+      if (this.encryptedColumns.isInternalColumn(column)) {
+        names.add(column.name);
+      }
+    }
+
+    return names;
+  }
+
+  private assertServerSide(): void {
+    if (typeof self !== 'undefined') {
+      throw new Error(
+        `Column encryption executes where the database driver runs (the server); the browser ` +
+          `must reach encrypted tables through DbService.`
+      );
+    }
+  }
+}

package/src/encryption/EncryptionTokenMaintenance.ts

@@ -0,0 +1,123 @@
+import { QueryBuilder } from '@proteinjs/db-query';
+import type { Table } from '../Table';
+import type { Record } from '../Record';
+import { EncryptedColumns, EncryptionSearchToken } from './EncryptedColumns';
+import { DataKeyStore } from './DataKeyStore';
+import { SearchTokenizer } from './SearchTokenizer';
+
+/**
+ * Search-token upkeep for `encrypted: { searchable: 'contains' }` columns: the same write
+ * that stores a row's ciphertext also derives the value's search tokens (words, trigrams,
+ * short prefixes — `SearchTokenizer`), fingerprints each with the row owner's index key,
+ * and maintains them as rows of the derived token table (`EncryptedColumns.tokenTableFor`)
+ * — the portable inverted index the query translator's candidate step reads.
+ *
+ * Invoked by `Db` around its DML (insert/update/delete), through a SYSTEM Db riding the
+ * same driver and ambient transaction — inside a transaction, token rows commit or roll
+ * back with the row; outside one, the token write follows the row write immediately.
+ */
+export class EncryptionTokenMaintenance {
+  private static readonly ID_CHUNK_SIZE = 500;
+  private encryptedColumns = new EncryptedColumns();
+  private tokenizer = new SearchTokenizer();
+
+  /** Write token rows for the contains-searchable columns present in an inserted record. */
+  async afterInsert(table: Table<any>, record: any, keyOwner: string, systemDb: TokenMaintenanceDb): Promise<void> {
+    const touched = this.touchedContainsProps(table, record);
+    if (touched.length === 0) {
+      return;
+    }
+
+    await this.writeTokenRows(table, [record.id], record, touched, keyOwner, systemDb);
+  }
+
+  /** Rewrite token rows of the touched contains-searchable columns for the updated row ids. */
+  async afterUpdate(
+    table: Table<any>,
+    recordIds: string[],
+    record: any,
+    keyOwner: string,
+    systemDb: TokenMaintenanceDb
+  ): Promise<void> {
+    const touched = this.touchedContainsProps(table, record);
+    if (touched.length === 0 || recordIds.length === 0) {
+      return;
+    }
+
+    const tokenTable = this.encryptedColumns.tokenTableFor(table)!;
+    const touchedColumnNames = touched.map((prop) => ((table.columns as any)[prop] as { name: string }).name);
+    for (const idsChunk of this.chunk(recordIds, EncryptionTokenMaintenance.ID_CHUNK_SIZE)) {
+      const deleteQb = new QueryBuilder<EncryptionSearchToken>(tokenTable.name)
+        .condition({ field: 'recordId', operator: 'IN', value: idsChunk })
+        .condition({ field: 'columnName', operator: 'IN', value: touchedColumnNames });
+      await systemDb.delete(tokenTable, deleteQb);
+    }
+
+    await this.writeTokenRows(table, recordIds, record, touched, keyOwner, systemDb);
+  }
+
+  /** Remove every token row of deleted records. */
+  async afterDelete(table: Table<any>, deletedIds: string[], systemDb: TokenMaintenanceDb): Promise<void> {
+    const tokenTable = this.encryptedColumns.tokenTableFor(table);
+    if (!tokenTable || deletedIds.length === 0) {
+      return;
+    }
+
+    for (const idsChunk of this.chunk(deletedIds, EncryptionTokenMaintenance.ID_CHUNK_SIZE)) {
+      const deleteQb = new QueryBuilder<EncryptionSearchToken>(tokenTable.name).condition({
+        field: 'recordId',
+        operator: 'IN',
+        value: idsChunk,
+      });
+      await systemDb.delete(tokenTable, deleteQb);
+    }
+  }
+
+  private async writeTokenRows(
+    table: Table<any>,
+    recordIds: string[],
+    record: any,
+    touchedProps: string[],
+    keyOwner: string,
+    systemDb: TokenMaintenanceDb
+  ): Promise<void> {
+    const tokenTable = this.encryptedColumns.tokenTableFor(table)!;
+    const writeKey = await new DataKeyStore().getWriteKey(keyOwner);
+    for (const prop of touchedProps) {
+      const value = record[prop];
+      if (value === null) {
+        continue; // null values carry no tokens
+      }
+
+      const columnName = ((table.columns as any)[prop] as { name: string }).name;
+      const fingerprints = this.tokenizer.fingerprints(this.tokenizer.tokensForValue(String(value)), writeKey.indexKey);
+      for (const recordId of recordIds) {
+        for (const token of fingerprints) {
+          await systemDb.insert(tokenTable, { recordId, columnName, token } as Omit<
+            EncryptionSearchToken,
+            keyof Record
+          >);
+        }
+      }
+    }
+  }
+
+  private touchedContainsProps(table: Table<any>, record: any): string[] {
+    return this.encryptedColumns.containsProps(table).filter((prop) => typeof record[prop] !== 'undefined');
+  }
+
+  private chunk<T>(items: T[], size: number): T[][] {
+    const chunks: T[][] = [];
+    for (let i = 0; i < items.length; i += size) {
+      chunks.push(items.slice(i, i + size));
+    }
+
+    return chunks;
+  }
+}
+
+/** The Db surface token maintenance needs (a system Db instance — see class doc). */
+export interface TokenMaintenanceDb {
+  insert<T extends Record>(table: Table<T>, record: Omit<T, keyof Record>): Promise<T>;
+  delete<T extends Record>(table: Table<T>, query: QueryBuilder<T>): Promise<number>;
+}

package/src/encryption/InMemoryMasterKeyProvider.ts

@@ -0,0 +1,42 @@
+import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'crypto';
+import { Base64Url } from './Base64Url';
+import { MasterKeyProvider } from './MasterKeyProvider';
+
+/**
+ * A `MasterKeyProvider` holding the master key in process memory. For tests and local
+ * development only — a production deployment keeps its master key in a vault
+ * (`GcpKmsMasterKeyProvider`), where the key material never enters the app process.
+ *
+ * Wraps with AES-256-GCM. Constructing with the same `secret` yields a provider that can
+ * unwrap keys wrapped by a previous instance (test fixtures across processes); constructing
+ * with no secret generates a random master key for the life of the process.
+ */
+export class InMemoryMasterKeyProvider implements MasterKeyProvider {
+  private masterKey: Buffer;
+
+  constructor(secret?: string) {
+    this.masterKey = secret ? createHash('sha256').update(secret).digest() : randomBytes(32);
+  }
+
+  async wrapDataKey(plaintextKeyMaterial: Buffer): Promise<string> {
+    const iv = randomBytes(12);
+    const cipher = createCipheriv('aes-256-gcm', this.masterKey, iv);
+    const ciphertext = Buffer.concat([cipher.update(plaintextKeyMaterial), cipher.final()]);
+    const authTag = cipher.getAuthTag();
+    return Base64Url.encode(Buffer.concat([iv, authTag, ciphertext]));
+  }
+
+  async unwrapDataKey(wrappedKeyMaterial: string): Promise<Buffer> {
+    const bytes = Base64Url.decode(wrappedKeyMaterial);
+    const iv = bytes.subarray(0, 12);
+    const authTag = bytes.subarray(12, 28);
+    const ciphertext = bytes.subarray(28);
+    const decipher = createDecipheriv('aes-256-gcm', this.masterKey, iv);
+    decipher.setAuthTag(authTag);
+    return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+  }
+
+  getMasterKeyId(): string {
+    return 'in-memory';
+  }
+}

package/src/encryption/MasterKeyProvider.ts

@@ -0,0 +1,26 @@
+/**
+ * The master-key seam of column encryption (see `EncryptedColumnConfig`).
+ *
+ * One master key exists per deployment, held in a key vault (Cloud KMS in production, an
+ * in-memory key in tests). The master key never encrypts row data directly — it wraps
+ * (encrypts) each user's randomly generated data key, and those per-user data keys encrypt
+ * the rows in that user's scope. The vault is therefore called once per data key
+ * (unwrap-once-per-key caching lives in `DataKeyStore`), never per row.
+ *
+ * Implementations:
+ * - `InMemoryMasterKeyProvider` (this package) — tests and local development.
+ * - `GcpKmsMasterKeyProvider` (`@proteinjs/db-encryption-driver-gcp-kms`) — Cloud KMS.
+ */
+export interface MasterKeyProvider {
+  /**
+   * Encrypt (wrap) raw data-key material under the master key.
+   * @returns an opaque string safe to store in the data-key table
+   */
+  wrapDataKey(plaintextKeyMaterial: Buffer): Promise<string>;
+
+  /** Decrypt (unwrap) previously wrapped data-key material. */
+  unwrapDataKey(wrappedKeyMaterial: string): Promise<Buffer>;
+
+  /** Stable identifier of the master key (a KMS resource name, or 'in-memory') — observability only. */
+  getMasterKeyId(): string;
+}

package/src/encryption/SearchTokenizer.ts

@@ -0,0 +1,103 @@
+import { createHmac } from 'crypto';
+import { Base64Url } from './Base64Url';
+
+/**
+ * Search tokens and keyed fingerprints for encrypted columns
+ * (`encrypted: { searchable: 'contains' | 'equality' }`).
+ *
+ * Definitions:
+ * - **Search token** — a normalized fragment of a text value: its lowercased words, each
+ *   word's trigrams (three-letter fragments), and each word's 1- and 2-letter prefixes
+ *   (so 1–2 character queries behave as word-prefix search).
+ * - **Keyed fingerprint (HMAC-SHA256)** — computable only with the owner's index key
+ *   (`DataKeyMaterial.indexKey`); without it, stored fingerprints can be neither created
+ *   nor reversed, so raw database access cannot probe ("does any title contain X?").
+ *
+ * Query-side cover: a candidate row must hold ALL of the query's fragment fingerprints.
+ * The cover can over-match (fragments present but not contiguous) and — for query words of
+ * ≥3 characters — can never under-match; the query layer verifies every candidate against
+ * the decrypted value before returning it (the pg_trgm recheck semantics), so results are
+ * exact. Documented narrowing: a 1–2 character query WORD matches word beginnings only
+ * (a 1–2 character mid-word substring is unsupported and behaves as prefix search).
+ *
+ * Equality fingerprints cover the EXACT serialized value (no normalization): `=` keeps its
+ * native case-sensitive semantics with no recheck, and a unique index on the fingerprint
+ * column enforces per-owner value uniqueness.
+ */
+export class SearchTokenizer {
+  /** All write-side tokens of `value` (deduplicated, un-fingerprinted). */
+  tokensForValue(value: string): string[] {
+    const tokens = new Set<string>();
+    for (const word of this.words(value)) {
+      tokens.add(`w:${word}`);
+      tokens.add(`p:${word.slice(0, 1)}`);
+      if (word.length >= 2) {
+        tokens.add(`p:${word.slice(0, 2)}`);
+      }
+      for (const trigram of this.trigrams(word)) {
+        tokens.add(`t:${trigram}`);
+      }
+    }
+
+    return Array.from(tokens);
+  }
+
+  /**
+   * The query-side fragment cover of `query`: for each query word, its trigrams (≥3 chars)
+   * or its prefix token (1–2 chars). Every fragment must match for a row to be a candidate.
+   */
+  fragmentsForQuery(query: string): string[] {
+    const fragments = new Set<string>();
+    for (const word of this.words(query)) {
+      if (word.length >= 3) {
+        for (const trigram of this.trigrams(word)) {
+          fragments.add(`t:${trigram}`);
+        }
+      } else {
+        fragments.add(`p:${word}`);
+      }
+    }
+
+    return Array.from(fragments);
+  }
+
+  /** Keyed fingerprint of one token. */
+  fingerprint(token: string, indexKey: Buffer): string {
+    return Base64Url.encode(createHmac('sha256', indexKey).update(token, 'utf8').digest());
+  }
+
+  fingerprints(tokens: string[], indexKey: Buffer): string[] {
+    return tokens.map((token) => this.fingerprint(token, indexKey));
+  }
+
+  /** Whole-value equality fingerprint — over the exact serialized value (see class doc). */
+  equalityFingerprint(value: string, indexKey: Buffer): string {
+    return Base64Url.encode(createHmac('sha256', indexKey).update(`eq:${value}`, 'utf8').digest());
+  }
+
+  /**
+   * The declared bounded reveal for `encrypted: { sortKey: { revealPrefix: N } }`: the
+   * normalized (lowercased) first N characters, stored beside the ciphertext for native
+   * ORDER BY. Raw database access can read those N characters — the declared, documented
+   * leak, chosen at the schema, never the default.
+   */
+  sortPrefix(value: string, revealPrefix: number): string {
+    return value.toLowerCase().slice(0, revealPrefix);
+  }
+
+  private words(text: string): string[] {
+    return text
+      .toLowerCase()
+      .split(/[^\p{L}\p{N}]+/u)
+      .filter((word) => word.length > 0);
+  }
+
+  private trigrams(word: string): string[] {
+    const trigrams: string[] = [];
+    for (let i = 0; i + 3 <= word.length; i++) {
+      trigrams.push(word.slice(i, i + 3));
+    }
+
+    return trigrams;
+  }
+}

package/src/schema/TableManager.ts

@@ -6,6 +6,8 @@ import { SchemaMetadata } from './SchemaMetadata';
 import { DbDriver } from '../Db';
 import { DynamicReferenceColumn, DynamicReferenceTableNameColumn } from '../Columns';
 import { StatementConfigFactory } from '../StatementConfigFactory';
+import { EncryptedColumns } from '../encryption/EncryptedColumns';
+import { findDbEncryptionConfig } from '../encryption/DbEncryptionConfig';
 
 /**
  * Thrown by the pre-sync duplicate check when a unique index is about to be added to an existing
@@ -75,7 +77,7 @@ export class TableManager {
    * just-created table sees it live.
    */
   async loadTables(): Promise<void> {
-    const tables = getTables();
+    const tables = this.prepareTablesForLoad(getTables());
     const absentTables: Table<any>[] = [];
     const existingTables: Table<any>[] = [];
     for (const table of tables) {
@@ -103,6 +105,12 @@ export class TableManager {
   }
 
   async loadTable(table: Table<any>): Promise<void> {
+    for (const preparedTable of this.prepareTablesForLoad([table])) {
+      await this.loadSingleTable(preparedTable);
+    }
+  }
+
+  private async loadSingleTable(table: Table<any>): Promise<void> {
     this.validateDynamicReferenceColumns(table);
 
     if (await this.tableExists(table)) {
@@ -118,6 +126,40 @@ export class TableManager {
     }
   }
 
+  /**
+   * Column-encryption schema pass (`ColumnOptions.encrypted`): validate declarations,
+   * inject the derived companion columns, and append each table's derived search-token
+   * table so it is created/altered alongside its base table. When the deployment turns on
+   * `requireEncryptedDeclarations` (or `DB_REQUIRE_ENCRYPTED_DECLARATIONS=true`), every
+   * text-holding column must declare `encrypted` — registration fails loudly here otherwise.
+   */
+  private prepareTablesForLoad(tables: Table<any>[]): Table<any>[] {
+    const encryptedColumns = new EncryptedColumns();
+    const requireDeclarations = this.requireEncryptedDeclarations();
+    const prepared: Table<any>[] = [];
+    for (const table of tables) {
+      encryptedColumns.ensureSchema(table);
+      if (requireDeclarations) {
+        encryptedColumns.validateDeclarations(table);
+      }
+      prepared.push(table);
+      const tokenTable = encryptedColumns.tokenTableFor(table);
+      if (tokenTable) {
+        prepared.push(tokenTable);
+      }
+    }
+
+    return prepared;
+  }
+
+  private requireEncryptedDeclarations(): boolean {
+    if (getEnvVar('DB_REQUIRE_ENCRYPTED_DECLARATIONS') === 'true') {
+      return true;
+    }
+
+    return !!findDbEncryptionConfig()?.requireEncryptedDeclarations;
+  }
+
   private async alterTableIfChanged(table: Table<any>): Promise<void> {
     const tableChanges = await this.getTableChanges(table);
     if (this.shouldAlterTable(tableChanges)) {