@flowerforce/flowerbase 1.7.5 → 1.7.6-beta.1

package/src/utils/__tests__/rule.test.ts

@@ -1,3 +1,4 @@
+import { ObjectId } from 'mongodb'
 import rulesMatcherUtils from '../rules-matcher/utils'
 
 describe('rule function', () => {
@@ -46,4 +47,42 @@ describe('rule function', () => {
       rulesMatcherUtils.rule(missingOperatorBlock, mockData, mockOptions)
     }).toThrow('Error missing operator:$notFoundOperator')
   })
+
+  it('should support %stringToOid with $ref values', () => {
+    const companyId = new ObjectId()
+    const data = {
+      user: {
+        _id: companyId
+      },
+      auth: {
+        company: companyId.toHexString()
+      }
+    }
+
+    const result = rulesMatcherUtils.rule({ _id: { '%stringToOid': '$ref:auth.company' } }, data, {
+      prefix: 'user'
+    })
+
+    expect(result.valid).toBe(true)
+    expect(result.name).toBe('user._id___%stringToOid')
+  })
+
+  it('should support %oidToString with $ref values', () => {
+    const authId = new ObjectId()
+    const data = {
+      user: {
+        authId: authId.toHexString()
+      },
+      auth: {
+        id: authId
+      }
+    }
+
+    const result = rulesMatcherUtils.rule({ authId: { '%oidToString': '$ref:auth.id' } }, data, {
+      prefix: 'user'
+    })
+
+    expect(result.valid).toBe(true)
+    expect(result.name).toBe('user.authId___%oidToString')
+  })
 })

package/src/utils/__tests__/rulesMatcherInterfaces.test.ts

@@ -22,6 +22,8 @@ describe('Enums and Types', () => {
     expect(RulesOperators.$nin).toBe('$nin')
     expect(RulesOperators.$all).toBe('$all')
     expect(RulesOperators.$regex).toBe('$regex')
+    expect(RulesOperators['%stringToOid']).toBe('%stringToOid')
+    expect(RulesOperators['%oidToString']).toBe('%oidToString')
   })
 
   it('should validate RulesOperatorsInArray type', () => {

package/src/utils/index.ts

@@ -1,5 +1,16 @@
-import fs from 'fs'
+import fs from 'node:fs'
+import path from 'node:path'
 
 export const readFileContent = (filePath: string) => fs.readFileSync(filePath, 'utf-8')
 export const readJsonContent = (filePath: string) =>
   JSON.parse(readFileContent(filePath)) as unknown
+
+export const recursivelyCollectFiles = (dir: string): string[] => {
+  return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+    const fullPath = path.join(dir, entry.name)
+    if (entry.isDirectory()) {
+      return recursivelyCollectFiles(fullPath)
+    }
+    return entry.isFile() ? [fullPath] : []
+  })
+}

package/src/utils/initializer/exposeRoutes.ts

@@ -2,7 +2,7 @@ import { uptime } from 'node:process'
 import { FastifyInstance } from 'fastify'
 import { RegistrationDto } from '../../auth/providers/local-userpass/dtos'
 import { AUTH_ENDPOINTS, REGISTRATION_SCHEMA } from '../../auth/utils'
-import { API_VERSION, AUTH_CONFIG, DB_NAME, DEFAULT_CONFIG } from '../../constants'
+import { API_VERSION, AUTH_CONFIG, AUTH_DB_NAME, DEFAULT_CONFIG } from '../../constants'
 import { PROVIDER } from '../../shared/models/handleUserRegistration.model'
 import { hashPassword } from '../crypto'
 
@@ -46,7 +46,7 @@ export const exposeRoutes = async (fastify: FastifyInstance) => {
       schema: REGISTRATION_SCHEMA
     }, async function (req, res) {
       const { authCollection } = AUTH_CONFIG
-      const db = fastify.mongo.client.db(DB_NAME)
+      const db = fastify.mongo.client.db(AUTH_DB_NAME)
       const { email, password } = req.body
       const hashedPassword = await hashPassword(password)
       const now = new Date()

package/src/utils/initializer/mongodbCSFLE.ts

@@ -0,0 +1,224 @@
+import {
+  ClientEncryption,
+  MongoClient,
+  UUID,
+  Binary,
+  type Document,
+  type AWSEncryptionKeyOptions,
+  type AWSKMSProviderConfiguration,
+  type AzureEncryptionKeyOptions,
+  type AzureKMSProviderConfiguration,
+  type GCPKMSProviderConfiguration,
+  type GCPEncryptionKeyOptions,
+  type KMIPKMSProviderConfiguration,
+  type KMIPEncryptionKeyOptions,
+  type LocalKMSProviderConfiguration,
+  type AutoEncryptionExtraOptions,
+  type KMSProviders,
+  type AutoEncryptionOptions,
+} from "mongodb";
+import { EncryptionSchemaProperty, MappedEncryptionSchema, MappedEncryptionSchemaProperty, type EncryptionSchemas } from "../../features/encryption/interface";
+import { DEFAULT_CONFIG } from "../../constants";
+
+type KMSProviderConfig =
+  | {
+    /**
+     * The alias of the key. It must be referenced in the schema map
+     * to select which key to use for encryption.
+     */
+    keyAlias: string
+    /**
+     * KMS Provider name.
+     */
+    provider: "aws"
+    /**
+     * KMS Provider specific authorization configuration.
+     */
+    config: AWSKMSProviderConfiguration
+    /**
+     * Configuration of the master key.
+     */
+    masterKey: AWSEncryptionKeyOptions
+  } | {
+    keyAlias: string
+    provider: "azure"
+    config: AzureKMSProviderConfiguration,
+    masterKey: AzureEncryptionKeyOptions
+  } | {
+    keyAlias: string
+    provider: "gcp"
+    config: GCPKMSProviderConfiguration,
+    masterKey: GCPEncryptionKeyOptions
+  } | {
+    keyAlias: string
+    provider: "kmip"
+    config: KMIPKMSProviderConfiguration,
+    masterKey: KMIPEncryptionKeyOptions
+  } | {
+    keyAlias: string
+    provider: "local"
+    config: LocalKMSProviderConfiguration,
+  }
+
+export type MongoDbEncryptionConfig = {
+  kmsProviders: KMSProviderConfig[],
+  /**
+   * The Key Vault database name
+   * @default encryption
+   */
+  keyVaultDb?: string
+  /**
+ * The Key Vault database collection
+ * @default __keyVault
+ */
+  keyVaultCollection?: string
+  extraOptions?: AutoEncryptionExtraOptions
+}
+
+/**
+ * @internal
+ */
+type RequiredConfig = Required<Omit<MongoDbEncryptionConfig, "extraOptions">>
+
+type DataKey = { dataKeyId: UUID, dataKeyAlias: string }
+
+async function ensureUniqueKeyAltNameIndex(db: ReturnType<MongoClient['db']>, config: RequiredConfig): Promise<void> {
+  await db.collection(config.keyVaultCollection).createIndex(
+    { keyAltNames: 1 },
+    {
+      unique: true,
+      partialFilterExpression: { keyAltNames: { $exists: true } },
+    }
+  );
+}
+
+/**
+ * Ensure provided KMS Providers DEK keys exist in the key vault. If not, they are created.
+ */
+async function ensureDataEncryptionKeys(
+  clientEncryption: ClientEncryption,
+  keyVaultDb: ReturnType<MongoClient['db']>,
+  config: RequiredConfig
+): Promise<DataKey[]> {
+  const keys: DataKey[] = []
+
+  for (const kmsProvider of config.kmsProviders) {
+    const existingKey = await keyVaultDb.collection(config.keyVaultCollection).findOne({
+      keyAltNames: kmsProvider.keyAlias,
+    });
+
+    if (existingKey?._id instanceof Binary) {
+      keys.push({ dataKeyId: existingKey._id, dataKeyAlias: kmsProvider.keyAlias })
+      continue
+    }
+
+    const dataKeyId = await clientEncryption.createDataKey(kmsProvider.provider, {
+      masterKey: "masterKey" in kmsProvider ? kmsProvider.masterKey : undefined,
+      keyAltNames: [kmsProvider.keyAlias],
+    });
+    console.log(`[MongoDB Encryption] Created new key with alias ${kmsProvider.keyAlias}`)
+    keys.push({ dataKeyId, dataKeyAlias: kmsProvider.keyAlias })
+  }
+
+  return keys
+}
+
+/**
+ * Recursively resolve key aliases in an encryption schema to their corresponding key IDs.
+ */
+const resolveKeyAliases = (schema: EncryptionSchemaProperty, dataKeys: DataKey[]): MappedEncryptionSchemaProperty => {
+  if ("encrypt" in schema) {
+    if (!schema.encrypt.keyAlias) {
+      return schema
+    }
+    const keyId = dataKeys.find(k => k.dataKeyAlias === schema.encrypt.keyAlias)?.dataKeyId
+    if (!keyId) {
+      throw new Error(`Key with alias ${schema.encrypt.keyAlias} could not be found in the Key Vault.`)
+    }
+    return {
+      encrypt: {
+        bsonType: schema.encrypt.bsonType,
+        algorithm: schema.encrypt.algorithm,
+        keyId: [keyId]
+      }
+    }
+  }
+  const mappedSchema: MappedEncryptionSchema = {
+    bsonType: "object",
+    properties: Object.entries(schema.properties).reduce((acc, [property, config]) => {
+      acc[property] = resolveKeyAliases(config, dataKeys)
+      return acc
+    }, {} as Record<string, MappedEncryptionSchemaProperty>)
+  }
+
+  if (schema.encryptMetadata) {
+    const keyId = dataKeys.find(k => k.dataKeyAlias === schema.encryptMetadata!.keyAlias)?.dataKeyId
+    if (!keyId) {
+      throw new Error(`Key with alias ${schema.encryptMetadata.keyAlias} could not be found in the Key Vault.`)
+    }
+    mappedSchema.encryptMetadata = { keyId: [keyId] }
+  }
+
+  return mappedSchema
+}
+
+export const buildSchemaMap = (schemas: EncryptionSchemas, dataKeys: DataKey[]) => {
+  return Object.entries(schemas).reduce((acc, [key, schema]) => {
+    acc[key] = resolveKeyAliases(schema, dataKeys)
+    return acc
+  }, {} as Record<string, Document>)
+}
+
+/**
+ * Setup MongoDB Client-Side Field Level Encryption (CSFLE).
+ * @see https://www.mongodb.com/docs/manual/core/csfle
+ */
+export const setupMongoDbCSFLE = async (
+  config: MongoDbEncryptionConfig & { mongodbUrl: string; schemas?: EncryptionSchemas }
+): Promise<AutoEncryptionOptions> => {
+  if (config.kmsProviders.length === 0) {
+    throw new Error('At least one KMS Provider is required when using MongoDB encryption')
+  }
+
+  const requiredConfig: RequiredConfig = {
+    kmsProviders: config.kmsProviders,
+    keyVaultDb: config.keyVaultDb ?? DEFAULT_CONFIG.MONGODB_ENCRYPTION_CONFIG.keyVaultDb,
+    keyVaultCollection: config.keyVaultDb ?? DEFAULT_CONFIG.MONGODB_ENCRYPTION_CONFIG.keyVaultCollection,
+  }
+
+  const kmsProviders = requiredConfig.kmsProviders.reduce(
+    (acc, { provider, config }) => ({ ...acc, [provider]: config }),
+    {} as KMSProviders
+  )
+
+  const keyVaultNamespace = `${requiredConfig.keyVaultDb}.${requiredConfig.keyVaultCollection}`
+  const keyVaultClient = new MongoClient(config.mongodbUrl, {
+    maxPoolSize: 1,
+    autoEncryption: {
+      keyVaultNamespace,
+      kmsProviders,
+      extraOptions: config.extraOptions
+    }
+  });
+
+  await keyVaultClient.connect();
+
+  const keyVaultDb = keyVaultClient.db(requiredConfig.keyVaultDb);
+  await ensureUniqueKeyAltNameIndex(keyVaultDb, requiredConfig)
+
+  const clientEncryption = new ClientEncryption(keyVaultClient, {
+    keyVaultNamespace,
+    kmsProviders,
+  });
+
+  const dataKeys = await ensureDataEncryptionKeys(clientEncryption, keyVaultDb, requiredConfig)
+
+  await keyVaultClient.close()
+
+  return {
+    keyVaultNamespace,
+    kmsProviders,
+    schemaMap: config.schemas ? buildSchemaMap(config.schemas, dataKeys) : undefined,
+    extraOptions: config.extraOptions
+  }
+}

package/src/utils/initializer/registerPlugins.ts

@@ -1,5 +1,5 @@
 import cors from '@fastify/cors'
-import fastifyMongodb from '@fastify/mongodb'
+import fastifyMongodb, { FastifyMongodbOptions } from '@fastify/mongodb'
 import { FastifyInstance } from 'fastify'
 import fastifyRawBody from 'fastify-raw-body'
 import { CorsConfig } from '../../'
@@ -10,7 +10,9 @@ import { customFunctionController } from '../../auth/providers/custom-function/c
 import { localUserPassController } from '../../auth/providers/local-userpass/controller'
 import { API_VERSION, DEFAULT_CONFIG } from '../../constants'
 import { Functions } from '../../features/functions/interface'
+import { EncryptionSchemas } from '../../features/encryption/interface'
 import monitoringPlugin from '../../monitoring/plugin'
+import { setupMongoDbCSFLE, MongoDbEncryptionConfig } from './mongodbCSFLE'
 
 type RegisterFunction = FastifyInstance['register']
 type RegisterParameters = Parameters<RegisterFunction>
@@ -21,6 +23,8 @@ type RegisterPluginsParams = {
   jwtSecret: string
   functionsList: Functions
   corsConfig?: CorsConfig
+  encryptionSchemas?: EncryptionSchemas
+  mongodbEncryptionConfig?: MongoDbEncryptionConfig
 }
 
 type RegisterConfig = {
@@ -41,14 +45,18 @@ export const registerPlugins = async ({
   mongodbUrl,
   jwtSecret,
   functionsList,
-  corsConfig
+  corsConfig,
+  mongodbEncryptionConfig,
+  encryptionSchemas
 }: RegisterPluginsParams) => {
   try {
     const registersConfig = await getRegisterConfig({
       mongodbUrl,
       jwtSecret,
       corsConfig,
-      functionsList
+      functionsList,
+      mongodbEncryptionConfig,
+      encryptionSchemas
     })
 
     registersConfig.forEach(({ plugin, options, pluginName }) => {
@@ -75,15 +83,23 @@ export const registerPlugins = async ({
 const getRegisterConfig = async ({
   mongodbUrl,
   jwtSecret,
-  corsConfig
-}: Pick<RegisterPluginsParams, 'jwtSecret' | 'mongodbUrl' | 'functionsList' | 'corsConfig'>): Promise<
-  RegisterConfig[]
-> => {
+  corsConfig,
+  encryptionSchemas,
+  mongodbEncryptionConfig,
+}: Omit<RegisterPluginsParams, "register">): Promise<RegisterConfig[]> => {
   const corsOptions = corsConfig ?? {
     origin: '*',
     methods: ['POST', 'GET']
   }
 
+  const autoEncryption = mongodbEncryptionConfig
+    ? await setupMongoDbCSFLE({
+      mongodbUrl,
+      schemas: encryptionSchemas,
+      ...mongodbEncryptionConfig
+    })
+    : undefined
+
   const baseConfig = [
     {
       pluginName: 'cors',
@@ -94,9 +110,28 @@ const getRegisterConfig = async ({
       pluginName: 'fastifyMongodb',
       plugin: fastifyMongodb,
       options: {
+        url: mongodbUrl,
         forceClose: true,
-        url: mongodbUrl
-      }
+        autoEncryption
+      } satisfies FastifyMongodbOptions
+    },
+    /**
+     * When auto-encryption is active, add another MongoDB client with bypass for change streams.
+     * The $changeStream operator does not support automatic encryption, only decryption.
+     * @see https://www.mongodb.com/docs/manual/core/csfle/reference/supported-operations
+     */
+    autoEncryption && {
+      pluginName: 'fastifyMongodb',
+      plugin: fastifyMongodb,
+      options: {
+        name: "changestream",
+        url: mongodbUrl,
+        forceClose: true,
+        autoEncryption: {
+          ...autoEncryption,
+          bypassAutoEncryption: true
+        }
+      } satisfies FastifyMongodbOptions
     },
     {
       pluginName: 'jwtAuthPlugin',
@@ -153,5 +188,5 @@ const getRegisterConfig = async ({
     } as RegisterConfig)
   }
 
-  return baseConfig
+  return baseConfig.filter(Boolean)
 }

package/src/utils/rules-matcher/interface.ts

@@ -331,6 +331,8 @@ export type Operators = {
    * @returns
    */
   $regex: OperatorsFunction
+  '%stringToOid': OperatorsFunction
+  '%oidToString': OperatorsFunction
 }
 
 export enum RulesOperators {
@@ -349,7 +351,9 @@ export enum RulesOperators {
   $nin = '$nin',
   $all = '$all',
   $size = '$size',
-  $regex = '$regex'
+  $regex = '$regex',
+  '%stringToOid' = '%stringToOid',
+  '%oidToString' = '%oidToString'
 }
 
 export type RulesOperatorsInArray<T> = Partial<{

package/src/utils/rules-matcher/utils.ts

@@ -1,9 +1,59 @@
+import { ObjectId } from 'bson'
 import _get from 'lodash/get'
-import _intersection from 'lodash/intersection'
 import _trimStart from 'lodash/trimStart'
 import { Operators, RulesMatcherUtils, RulesObject } from './interface'
 
 const EMPTY_STRING_REGEXP = /^\s*$/
+const HEX_24_REGEXP = /^[a-fA-F0-9]{24}$/
+
+const toObjectIdHex = (value: unknown): string | null => {
+  if (value instanceof ObjectId) {
+    return value.toHexString()
+  }
+
+  if (typeof value === 'string') {
+    if (!HEX_24_REGEXP.test(value) || !ObjectId.isValid(value)) {
+      return null
+    }
+    return new ObjectId(value).toHexString()
+  }
+
+  if (!value || typeof value !== 'object') {
+    return null
+  }
+
+  const maybeObjectId = value as { _bsontype?: string; toHexString?: () => string }
+  if (maybeObjectId._bsontype === 'ObjectId' && typeof maybeObjectId.toHexString === 'function') {
+    const hex = maybeObjectId.toHexString()
+    return HEX_24_REGEXP.test(hex) ? hex.toLowerCase() : null
+  }
+
+  return null
+}
+
+const areSemanticallyEqual = (left: unknown, right: unknown): boolean => {
+  const leftOid = toObjectIdHex(left)
+  const rightOid = toObjectIdHex(right)
+
+  if (leftOid || rightOid) {
+    return leftOid !== null && rightOid !== null && leftOid === rightOid
+  }
+
+  return left === right
+}
+
+const includesWithSemanticEquality = (value: unknown, candidate: unknown): boolean =>
+  rulesMatcherUtils
+    .forceArray(candidate)
+    .some((item) =>
+      rulesMatcherUtils
+        .forceArray(value)
+        .some((sourceItem) =>
+          rulesMatcherUtils.forceArray(item).some((candidateItem) =>
+            areSemanticallyEqual(sourceItem, candidateItem)
+          )
+        )
+    )
 
 /**
  * Defines a utility object named rulesMatcherUtils, which contains various helper functions used for processing rules and data in a rule-matching context.
@@ -24,10 +74,10 @@ const rulesMatcherUtils: RulesMatcherUtils = {
     const valueRef =
       value && String(value).indexOf('$ref:') === 0
         ? _get(
-            data,
-            rulesMatcherUtils.getPath(value.replace('$ref:', ''), prefix),
-            undefined
-          )
+          data,
+          rulesMatcherUtils.getPath(value.replace('$ref:', ''), prefix),
+          undefined
+        )
         : value
 
     if (!operators[op]) {
@@ -230,9 +280,9 @@ const rulesMatcherUtils: RulesMatcherUtils = {
 export const operators: Operators = {
   $exists: (a, b) => !rulesMatcherUtils.isEmpty(a) === b,
 
-  $eq: (a, b) => a === b,
+  $eq: (a, b) => areSemanticallyEqual(a, b),
 
-  $ne: (a, b) => a !== b,
+  $ne: (a, b) => !areSemanticallyEqual(a, b),
 
   $gt: (a, b) => rulesMatcherUtils.forceNumber(a) > parseFloat(b),
 
@@ -250,39 +300,35 @@ export const operators: Operators = {
 
   $strLte: (a, b) => String(a || '').length <= parseFloat(b),
 
-  $in: (a, b) =>
-    rulesMatcherUtils
-      .forceArray(b)
-      .some(
-        (c) =>
-          _intersection(rulesMatcherUtils.forceArray(a), rulesMatcherUtils.forceArray(c))
-            .length
-      ),
-
-  $nin: (a, b) =>
-    !rulesMatcherUtils
-      .forceArray(b)
-      .some(
-        (c) =>
-          _intersection(rulesMatcherUtils.forceArray(a), rulesMatcherUtils.forceArray(c))
-            .length
-      ),
+  $in: (a, b) => includesWithSemanticEquality(a, b),
+
+  $nin: (a, b) => !includesWithSemanticEquality(a, b),
 
   $all: (a, b) =>
-    rulesMatcherUtils
-      .forceArray(b)
-      .every(
-        (c) =>
-          _intersection(rulesMatcherUtils.forceArray(a), rulesMatcherUtils.forceArray(c))
-            .length
-      ),
+    rulesMatcherUtils.forceArray(b).every((candidate) =>
+      rulesMatcherUtils
+        .forceArray(a)
+        .some((value) =>
+          rulesMatcherUtils.forceArray(candidate).some((item) => areSemanticallyEqual(value, item))
+        )
+    ),
 
   $size: (a, b) => Array.isArray(a) && a.length === parseFloat(b),
 
   $regex: (a, b, opt) =>
     rulesMatcherUtils
       .forceArray(b)
-      .some((c) => (c instanceof RegExp ? c.test(a) : new RegExp(c, opt).test(a)))
+      .some((c) => (c instanceof RegExp ? c.test(a) : new RegExp(c, opt).test(a))),
+
+  '%stringToOid': (a, b) => {
+    const converted = toObjectIdHex(b)
+    return converted !== null && areSemanticallyEqual(a, converted)
+  },
+
+  '%oidToString': (a, b) => {
+    const converted = toObjectIdHex(b)
+    return converted !== null && areSemanticallyEqual(a, converted)
+  }
 }
 
 // export default operators