@friggframework/core 2.0.0--canary.651.bc0e60a.0 → 2.0.0--canary.651.8ef5a7a.0

package/CLAUDE.md

@@ -224,18 +224,39 @@ packages/core/
   'asc' | 'desc' }`, nulls last, ties broken by id in the same direction;
   without it rows come in id order. `take` is 1–500. `omit` lists top-level
   mapping keys to leave out of the rows; never write such rows back. Path
-  segments must match `^[A-Za-z_][A-Za-z0-9_]*$`. **PostgreSQL only**: the
-  MongoDB, DocumentDB and legacy repositories inherit the port's
-  "not supported by this database adapter yet" error. It also refuses to run
-  while field-level encryption still encrypts `IntegrationMapping.mapping` on
-  write (see `database/encryption/README.md`). Validation lives in
-  `integration-mapping-query.js`; a new operator is one entry in its
-  `OPERATORS` table and one in the Postgres adapter's `CONDITION_SQL`.
-  **Cost**: one SQL statement per call, which reads every row of the
-  integration and evaluates the JSON paths per row, because no JSON index
-  exists. With ~4 KB mappings on PostgreSQL 16 that is about 0.3 s per call
-  at 10⁴ rows per integration and 2.5–3 s at 10⁵; a deep offset or an empty
-  page past the end costs about the same as the first page.
+  segments must match `^[A-Za-z_][A-Za-z0-9_]*$`. The PostgreSQL, MongoDB and
+  DocumentDB adapters give the same pages and totals;
+  `integration-mapping-repository-query-parity.test.js` checks that against
+  real databases when `QUERY_MAPPINGS_PARITY_MONGO_URL` /
+  `QUERY_MAPPINGS_PARITY_POSTGRES_URL` are set. Two orderings still differ:
+  strings compare by the database collation on PostgreSQL and by code point
+  on MongoDB and DocumentDB, and arrays or objects at the sort path order
+  among themselves only on PostgreSQL (the other adapters fall back to id).
+  The legacy `IntegrationMappingRepository` inherits the port's
+  "not supported by this database adapter yet" error. Every adapter refuses
+  to run while field-level encryption still encrypts
+  `IntegrationMapping.mapping` on write (see `database/encryption/README.md`).
+  Validation lives in `integration-mapping-query.js`; a new operator is one
+  entry in its `OPERATORS` table, one in the Postgres adapter's
+  `CONDITION_SQL` and one in `CONDITION_EXPRESSIONS` in
+  `integration-mapping-query-pipeline.js`, the aggregation stages both
+  MongoDB-protocol adapters share. Those stages may only use what Amazon
+  DocumentDB 4.0 and 5.0 support (no `$facet`, `$getField`, `$set` or
+  `$unset`).
+  **Cost**: PostgreSQL answers in one SQL statement, which reads every row
+  of the integration and evaluates the JSON paths per row, because no JSON
+  index exists. With ~4 KB mappings on PostgreSQL 16 that is about 0.3 s per
+  call at 10⁴ rows per integration and 2.5–3 s at 10⁵. MongoDB answers in one
+  aggregate: the `integrationId` index narrows `$match` to the integration,
+  the `$expr` is then evaluated per document, and `$facet` returns the page
+  with its `$count`, so a page (after `omit`) must fit the 16 MB document
+  limit. DocumentDB has no `$facet`: the page and the count are two
+  concurrent aggregates, so the total does not come from the same snapshot
+  as the page. Both sort in memory on a computed key (`allowDiskUse`). With
+  ~4 KB mappings on MongoDB 7 that is about 0.05–0.1 s per call at 10⁴ rows
+  per integration and 0.5–0.7 s at 10⁵ (the DocumentDB pipeline, run on
+  MongoDB, 0.6–0.8 s). On every adapter a deep offset or an empty page past
+  the end costs about the same as the first page.
 - `process-repository-*.js` - Process (long-running job) persistence.
   Implements `applyProcessUpdate(processId, ops)` — a race-safe alternative
   to `update(id, patch)` that routes increments, sets, and bounded-array

package/database/documentdb-encryption-service.js

@@ -1,5 +1,9 @@
 const { Cryptor } = require('../encrypt/Cryptor');
-const { getEncryptedFields, loadCustomEncryptionSchema } = require('./encryption/encryption-schema-registry');
+const {
+    getEncryptedFields,
+    getFieldsToEncryptOnWrite,
+    loadCustomEncryptionSchema,
+} = require('./encryption/encryption-schema-registry');
 
 /**
  * Encryption service specifically for DocumentDB repositories
@@ -107,7 +111,7 @@ class DocumentDBEncryptionService {
         }
 
         // Get encrypted fields from registry
-        const encryptedFieldsConfig = getEncryptedFields(modelName);
+        const encryptedFieldsConfig = getFieldsToEncryptOnWrite(modelName);
         if (!encryptedFieldsConfig || encryptedFieldsConfig.length === 0) {
             return document;
         }

package/database/documentdb-utils.js

@@ -166,11 +166,12 @@ async function findManyDrained(client, collection, filter = {}, options = {}) {
     return drainCursor(client, collection, first);
 }
 
-async function aggregateDrained(client, collection, pipeline) {
+async function aggregateDrained(client, collection, pipeline, options = {}) {
     const first = await client.$runCommandRaw({
         aggregate: collection,
         pipeline,
         cursor: { batchSize: DRAIN_BATCH_SIZE },
+        ...options,
     });
     return drainCursor(client, collection, first);
 }

package/database/encryption/README.md

@@ -814,11 +814,12 @@ export AES_KEY=$(openssl rand -hex 16)  # Generate 32-char key
 ❌ Query on encrypted fields (not supported)
 ❌ Manually decrypt data (use extension)
 
-`IntegrationMappingRepository.queryMappings()` (PostgreSQL) filters and sorts
-inside the `mapping` JSON, so it refuses to run while field-level encryption
-still encrypts `IntegrationMapping.mapping` on write. Opt the field out in the
-app definition, together with any nested `mapping.*` path that
-`encryption.schema` encrypts:
+The integration mapping repositories' `queryMappings()` (PostgreSQL, MongoDB
+and DocumentDB) filters and sorts inside the `mapping` JSON, so it refuses to
+run while field-level encryption still encrypts `IntegrationMapping.mapping` on
+write. Opt the field out in the app definition, together with any nested
+`mapping.*` path that `encryption.schema` encrypts; the Prisma extension and
+`DocumentDBEncryptionService` both honor the opt-out:
 
 ```javascript
 encryption: {

package/database/encryption/integration-mapping-encryption.js

@@ -21,6 +21,23 @@ function getMappingFieldsEncryptedOnWrite() {
     return fields;
 }
 
+/**
+ * @throws {Error} When field-level encryption still encrypts `mapping`, or a
+ *   nested `mapping.*` path, on write, naming the opt-out that lifts it
+ */
+function assertMappingWrittenUnencrypted() {
+    const fields = getMappingFieldsEncryptedOnWrite();
+    if (fields.length === 0) return;
+
+    const encrypted = fields
+        .map((field) => `IntegrationMapping.${field}`)
+        .join(', ');
+    const optOut = fields.map((field) => `'${field}'`).join(', ');
+    throw new Error(
+        `queryMappings: field-level encryption still encrypts ${encrypted} on write, so it cannot be queried. Opt out by adding ${optOut} to appDefinition.encryption.disable.IntegrationMapping.`
+    );
+}
+
 function encryptedMappingFields() {
     if (!getEncryptionConfig().enabled) return [];
 
@@ -36,6 +53,7 @@ function resetMappingEncryptionCheck() {
 }
 
 module.exports = {
+    assertMappingWrittenUnencrypted,
     getMappingFieldsEncryptedOnWrite,
     resetMappingEncryptionCheck,
 };

package/integrations/repositories/integration-mapping-query-pipeline.js

@@ -0,0 +1,152 @@
+const SORT_KEY = '__sortKey';
+const MONGO_DIRECTIONS = { asc: 1, desc: -1 };
+const SORT_TYPE_RANKS = [
+    ['string'],
+    ['int', 'long', 'double', 'decimal'],
+    ['bool'],
+    ['array'],
+    ['object'],
+];
+const SCALAR_TYPES = SORT_TYPE_RANKS.slice(0, 3).flat();
+
+const isType = (expression, type) => ({ $eq: [{ $type: expression }, type] });
+const isAbsent = (expression) => ({
+    $eq: [{ $ifNull: [expression, null] }, null],
+});
+const mappingField = (segments) => `$mapping.${segments.join('.')}`;
+
+const CONDITION_EXPRESSIONS = {
+    exists: (operand) => ({ $ne: [{ $ifNull: [operand, null] }, null] }),
+    notExists: (operand) => isAbsent(operand),
+    in: (operand, values) => ({
+        $and: [
+            isType(operand, 'string'),
+            { $in: [operand, { $literal: values }] },
+        ],
+    }),
+    notStartsWith: (operand, prefix) => ({
+        $cond: [
+            isType(operand, 'string'),
+            {
+                $ne: [
+                    { $substrCP: [operand, 0, [...prefix].length] },
+                    { $literal: prefix },
+                ],
+            },
+            true,
+        ],
+    }),
+};
+
+/**
+ * Aggregation stages that answer a validated queryMappings query on the
+ * MongoDB wire protocol.
+ *
+ * @param {*} integrationId - The value IntegrationMapping.integrationId is stored as
+ * @param {ReturnType<import('./integration-mapping-query').validateMappingQuery>} query
+ * @returns {{match: Object, page: Object[], sort: Object}} `match` selects
+ *   the matching documents; `page` sorts, skips and limits them; `sort` is
+ *   the $sort stage inside `page`
+ */
+function buildMappingQueryStages(
+    integrationId,
+    { where, orderBy, skip, take, omit }
+) {
+    const match = {
+        $match: {
+            integrationId,
+            $expr: {
+                $and: [
+                    isType('$mapping', 'object'),
+                    ...where.map(entryExpression),
+                ],
+            },
+        },
+    };
+    const sort = { $sort: sortSpec(orderBy) };
+    const page = [
+        ...(orderBy
+            ? [{ $addFields: { [SORT_KEY]: sortKey(orderBy.path) } }]
+            : []),
+        ...(omit.length > 0 ? [{ $project: omitProjection(omit) }] : []),
+        sort,
+        ...(skip > 0 ? [{ $skip: skip }] : []),
+        { $limit: take },
+    ];
+    return { match, page, sort };
+}
+
+function entryExpression(entry) {
+    if (!entry.anyOf) return conditionExpression(entry);
+    return { $or: entry.anyOf.map(conditionExpression) };
+}
+
+function conditionExpression({ field, path, op, value }) {
+    const operand = field === 'sourceId' ? '$sourceId' : resolvePath(path);
+    return CONDITION_EXPRESSIONS[op](operand, value);
+}
+
+/**
+ * The value at a mapping path; null when an enclosing value is not an object.
+ */
+function resolvePath(segments) {
+    const enclosing = segments
+        .slice(0, -1)
+        .map((_, i) =>
+            isType(mappingField(segments.slice(0, i + 1)), 'object')
+        );
+    if (enclosing.length === 0) return mappingField(segments);
+    return {
+        $cond: [
+            enclosing.length === 1 ? enclosing[0] : { $and: enclosing },
+            mappingField(segments),
+            null,
+        ],
+    };
+}
+
+/**
+ * Orders values like jsonb: strings < numbers < booleans < arrays < objects,
+ * scalars by value within their type. Null and absent values sort last in
+ * both directions.
+ */
+function sortKey(path) {
+    const type = { $type: '$$value' };
+    return {
+        $let: {
+            vars: { value: resolvePath(path) },
+            in: {
+                missing: isAbsent('$$value'),
+                rank: {
+                    $switch: {
+                        branches: SORT_TYPE_RANKS.map((types, i) => ({
+                            case: { $in: [type, types] },
+                            then: i + 1,
+                        })),
+                        default: 0,
+                    },
+                },
+                value: {
+                    $cond: [{ $in: [type, SCALAR_TYPES] }, '$$value', null],
+                },
+            },
+        },
+    };
+}
+
+function sortSpec(orderBy) {
+    if (!orderBy) return { _id: 1 };
+    const direction = MONGO_DIRECTIONS[orderBy.direction];
+    return {
+        [`${SORT_KEY}.missing`]: 1,
+        [`${SORT_KEY}.rank`]: direction,
+        [`${SORT_KEY}.value`]: direction,
+        _id: direction,
+    };
+}
+
+function omitProjection(omit) {
+    return Object.fromEntries(omit.map((key) => [`mapping.${key}`, 0]));
+}
+
+module.exports = { buildMappingQueryStages };

package/integrations/repositories/integration-mapping-repository-documentdb.js

@@ -8,6 +8,7 @@ const {
     updateOne,
     deleteOne,
     deleteMany,
+    aggregate,
     aggregateDrained,
 } = require('../../database/documentdb-utils');
 const {
@@ -16,6 +17,20 @@ const {
 const {
     DocumentDBEncryptionService,
 } = require('../../database/documentdb-encryption-service');
+const {
+    assertMappingWrittenUnencrypted,
+} = require('../../database/encryption/integration-mapping-encryption');
+const { validateMappingQuery } = require('./integration-mapping-query');
+const {
+    buildMappingQueryStages,
+} = require('./integration-mapping-query-pipeline');
+
+function storedIntegrationId(id) {
+    if (!['string', 'number'].includes(typeof id) || id === '') {
+        throw new TypeError(`Invalid ID: ${id}`);
+    }
+    return String(id);
+}
 
 class IntegrationMappingRepositoryDocumentDB extends IntegrationMappingRepositoryInterface {
     constructor() {
@@ -179,6 +194,44 @@ class IntegrationMappingRepositoryDocumentDB extends IntegrationMappingRepositor
         return decryptedDocs.map((doc) => this._mapMapping(doc));
     }
 
+    /**
+     * @param {string} integrationId
+     * @param {Object} query - See IntegrationMappingRepositoryInterface.queryMappings
+     * @returns {Promise<{mappings: Array<Object>, total: number}>}
+     */
+    async queryMappings(integrationId, query) {
+        const validated = validateMappingQuery(query);
+        const stored = storedIntegrationId(integrationId);
+        assertMappingWrittenUnencrypted();
+        const { match, page, sort } = buildMappingQueryStages(
+            stored,
+            validated
+        );
+
+        const [docs, counts] = await Promise.all([
+            aggregateDrained(
+                this.prisma,
+                'IntegrationMapping',
+                [match, ...page, sort],
+                { allowDiskUse: true }
+            ),
+            aggregate(this.prisma, 'IntegrationMapping', [
+                match,
+                { $count: 'total' },
+            ]),
+        ]);
+        const decryptedDocs = await Promise.all(
+            docs.map((doc) =>
+                this.encryptionService.decryptFields('IntegrationMapping', doc)
+            )
+        );
+
+        return {
+            mappings: decryptedDocs.map((doc) => this._mapMapping(doc)),
+            total: counts[0]?.total ?? 0,
+        };
+    }
+
     async deleteMapping(integrationId, sourceId) {
         const filter = this._compositeFilter(integrationId, sourceId);
         const result = await deleteOne(

package/integrations/repositories/integration-mapping-repository-interface.js

@@ -79,13 +79,17 @@ class IntegrationMappingRepositoryInterface {
      *   conditions, anyOf members included.
      * @param {{path: string, direction: 'asc'|'desc'}} [query.orderBy] - A
      *   mapping path; nulls last, ties broken by id in the same direction.
-     *   Without it rows are ordered by id ascending.
+     *   Values order string < number < boolean < array < object. Strings
+     *   compare by the database collation on PostgreSQL and by code point on
+     *   MongoDB and DocumentDB; arrays and objects order among themselves
+     *   only on PostgreSQL. Without it rows are ordered by id ascending.
      * @param {number} [query.skip=0] - Rows to skip (integer ≥ 0)
      * @param {number} query.take - Page size (integer 1–500)
      * @param {string[]} [query.omit=[]] - Top-level mapping keys to leave out of
      *   the returned rows; such projected rows must not be written back
      * @returns {Promise<{mappings: Array<Object>, total: number}>} The page, and
-     *   the number of rows matching `where`
+     *   the number of rows matching `where` (counted by a separate command on
+     *   DocumentDB, so not from the page's snapshot)
      */
     async queryMappings(integrationId, query) {
         throw new Error(

package/integrations/repositories/integration-mapping-repository-mongo.js

@@ -1,7 +1,25 @@
 const { prisma } = require('../../database/prisma');
+const {
+    assertMappingWrittenUnencrypted,
+} = require('../../database/encryption/integration-mapping-encryption');
 const {
     IntegrationMappingRepositoryInterface,
 } = require('./integration-mapping-repository-interface');
+const { validateMappingQuery } = require('./integration-mapping-query');
+const {
+    buildMappingQueryStages,
+} = require('./integration-mapping-query-pipeline');
+
+const OBJECT_ID_REGEX = /^[0-9a-fA-F]{24}$/;
+
+const fromRawDate = (raw) => new Date(raw.$date);
+
+function strictObjectId(id) {
+    if (typeof id !== 'string' || !OBJECT_ID_REGEX.test(id)) {
+        throw new TypeError(`Invalid ID: ${id} is not an ObjectId`);
+    }
+    return id;
+}
 
 /**
  * MongoDB Integration Mapping Repository Adapter
@@ -155,6 +173,57 @@ class IntegrationMappingRepositoryMongo extends IntegrationMappingRepositoryInte
         return counts;
     }
 
+    /**
+     * @param {string} integrationId
+     * @param {Object} query - See IntegrationMappingRepositoryInterface.queryMappings
+     * @returns {Promise<{mappings: Array<Object>, total: number}>}
+     */
+    async queryMappings(integrationId, query) {
+        const validated = validateMappingQuery(query);
+        const objectId = strictObjectId(integrationId);
+        assertMappingWrittenUnencrypted();
+        const { match, page } = buildMappingQueryStages(
+            { $oid: objectId },
+            validated
+        );
+
+        const result = await this.prisma.$runCommandRaw({
+            aggregate: 'IntegrationMapping',
+            pipeline: [
+                match,
+                {
+                    $facet: {
+                        mappings: page,
+                        total: [{ $count: 'total' }],
+                    },
+                },
+            ],
+            cursor: {},
+            allowDiskUse: true,
+        });
+        const [{ mappings, total }] = result.cursor.firstBatch;
+
+        return {
+            mappings: mappings.map((doc) => this._fromRawMapping(doc)),
+            total: total[0]?.total ?? 0,
+        };
+    }
+
+    /**
+     * A raw aggregate document, in the shape findMappingsByIntegration returns.
+     * @private
+     */
+    _fromRawMapping(doc) {
+        return {
+            id: doc._id.$oid,
+            integrationId: doc.integrationId.$oid,
+            sourceId: doc.sourceId ?? null,
+            mapping: doc.mapping ?? null,
+            createdAt: fromRawDate(doc.createdAt),
+            updatedAt: fromRawDate(doc.updatedAt),
+        };
+    }
+
     /**
      * Find mapping by ID
      * @param {string} id - Mapping ID

package/integrations/repositories/integration-mapping-repository-postgres.js

@@ -1,6 +1,6 @@
 const { prisma } = require('../../database/prisma');
 const {
-    getMappingFieldsEncryptedOnWrite,
+    assertMappingWrittenUnencrypted,
 } = require('../../database/encryption/integration-mapping-encryption');
 const {
     IntegrationMappingRepositoryInterface,
@@ -247,7 +247,7 @@ class IntegrationMappingRepositoryPostgres extends IntegrationMappingRepositoryI
         const { where, orderBy, skip, take, omit } =
             validateMappingQuery(query);
         const intIntegrationId = strictIntId(integrationId);
-        this._assertMappingWrittenUnencrypted();
+        assertMappingWrittenUnencrypted();
 
         const params = [];
         const bind = (v) => {
@@ -294,20 +294,6 @@ class IntegrationMappingRepositoryPostgres extends IntegrationMappingRepositoryI
         };
     }
 
-    /** @private */
-    _assertMappingWrittenUnencrypted() {
-        const fields = getMappingFieldsEncryptedOnWrite();
-        if (fields.length === 0) return;
-
-        const encrypted = fields
-            .map((field) => `IntegrationMapping.${field}`)
-            .join(', ');
-        const optOut = fields.map((field) => `'${field}'`).join(', ');
-        throw new Error(
-            `queryMappings: field-level encryption still encrypts ${encrypted} on write, so it cannot be queried. Opt out by adding ${optOut} to appDefinition.encryption.disable.IntegrationMapping.`
-        );
-    }
-
     /**
      * One where entry: a condition, or an anyOf group as a parenthesized OR.
      * @private

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@friggframework/core",
     "prettier": "@friggframework/prettier-config",
-    "version": "2.0.0--canary.651.bc0e60a.0",
+    "version": "2.0.0--canary.651.8ef5a7a.0",
     "dependencies": {
         "@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
         "@aws-sdk/client-kms": "^3.588.0",
@@ -48,9 +48,9 @@
         }
     },
     "devDependencies": {
-        "@friggframework/eslint-config": "2.0.0--canary.651.bc0e60a.0",
-        "@friggframework/prettier-config": "2.0.0--canary.651.bc0e60a.0",
-        "@friggframework/test": "2.0.0--canary.651.bc0e60a.0",
+        "@friggframework/eslint-config": "2.0.0--canary.651.8ef5a7a.0",
+        "@friggframework/prettier-config": "2.0.0--canary.651.8ef5a7a.0",
+        "@friggframework/test": "2.0.0--canary.651.8ef5a7a.0",
         "@prisma/client": "^6.19.3",
         "@types/lodash": "4.17.15",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
@@ -90,5 +90,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "bc0e60a39a4ee32316d346a02ef8da77d4d41b1f"
+    "gitHead": "8ef5a7a4c048da70b48afa1469fe63a71d9f1240"
 }