@classytic/mongokit 1.0.1 → 2.0.0

package/src/index.js

@@ -1,60 +1,71 @@
-/**
- * Repository Pattern - Data Access Layer
- * 
- * Event-driven, plugin-based abstraction for MongoDB operations
- * Inspired by Meta & Stripe's repository patterns
- * 
- * @module common/repositories
- * 
- * Documentation:
- * - README.md - Main documentation (concise overview)
- * - QUICK_REFERENCE.md - One-page cheat sheet
- * - EXAMPLES.md - Detailed examples and patterns
- */
-
-/**
- * MongoKit - Event-driven repository pattern for MongoDB
- * 
- * @module @classytic/mongokit
- * @author Sadman Chowdhury (Github: @siam923)
- * @license MIT
- */
-
-export { Repository } from './Repository.js';
-
-// Plugins
-export { fieldFilterPlugin } from './plugins/field-filter.plugin.js';
-export { timestampPlugin } from './plugins/timestamp.plugin.js';
-export { auditLogPlugin } from './plugins/audit-log.plugin.js';
-export { softDeletePlugin } from './plugins/soft-delete.plugin.js';
-export { methodRegistryPlugin } from './plugins/method-registry.plugin.js';
-export {
-  validationChainPlugin,
-  blockIf,
-  requireField,
-  autoInject,
-  immutableField,
-  uniqueField,
-} from './plugins/validation-chain.plugin.js';
-export { mongoOperationsPlugin } from './plugins/mongo-operations.plugin.js';
-export { batchOperationsPlugin } from './plugins/batch-operations.plugin.js';
-export { aggregateHelpersPlugin } from './plugins/aggregate-helpers.plugin.js';
-export { subdocumentPlugin } from './plugins/subdocument.plugin.js';
-
-// Utilities
-export {
-  getFieldsForUser,
-  getMongooseProjection,
-  filterResponseData,
-  createFieldPreset,
-} from './utils/field-selection.js';
-
-export * as actions from './actions/index.js';
-
-import { Repository } from './Repository.js';
-
-export const createRepository = (Model, plugins = []) => {
-  return new Repository(Model, plugins);
-};
-
-export default Repository;
+/**
+ * Repository Pattern - Data Access Layer
+ * 
+ * Event-driven, plugin-based abstraction for MongoDB operations
+ * Inspired by Meta & Stripe's repository patterns
+ * 
+ * @module common/repositories
+ * 
+ * Documentation:
+ * - README.md - Main documentation (concise overview)
+ * - QUICK_REFERENCE.md - One-page cheat sheet
+ * - EXAMPLES.md - Detailed examples and patterns
+ */
+
+/**
+ * MongoKit - Event-driven repository pattern for MongoDB
+ * 
+ * @module @classytic/mongokit
+ * @author Sadman Chowdhury (Github: @siam923)
+ * @license MIT
+ */
+
+/**
+ * @typedef {import('./pagination/PaginationEngine.js').PaginationConfig} PaginationConfig
+ * @typedef {import('./pagination/PaginationEngine.js').OffsetPaginationOptions} OffsetPaginationOptions
+ * @typedef {import('./pagination/PaginationEngine.js').KeysetPaginationOptions} KeysetPaginationOptions
+ * @typedef {import('./pagination/PaginationEngine.js').AggregatePaginationOptions} AggregatePaginationOptions
+ * @typedef {import('./pagination/PaginationEngine.js').OffsetPaginationResult} OffsetPaginationResult
+ * @typedef {import('./pagination/PaginationEngine.js').KeysetPaginationResult} KeysetPaginationResult
+ * @typedef {import('./pagination/PaginationEngine.js').AggregatePaginationResult} AggregatePaginationResult
+ */
+
+export { Repository } from './Repository.js';
+export { PaginationEngine } from './pagination/PaginationEngine.js';
+
+// Plugins
+export { fieldFilterPlugin } from './plugins/field-filter.plugin.js';
+export { timestampPlugin } from './plugins/timestamp.plugin.js';
+export { auditLogPlugin } from './plugins/audit-log.plugin.js';
+export { softDeletePlugin } from './plugins/soft-delete.plugin.js';
+export { methodRegistryPlugin } from './plugins/method-registry.plugin.js';
+export {
+  validationChainPlugin,
+  blockIf,
+  requireField,
+  autoInject,
+  immutableField,
+  uniqueField,
+} from './plugins/validation-chain.plugin.js';
+export { mongoOperationsPlugin } from './plugins/mongo-operations.plugin.js';
+export { batchOperationsPlugin } from './plugins/batch-operations.plugin.js';
+export { aggregateHelpersPlugin } from './plugins/aggregate-helpers.plugin.js';
+export { subdocumentPlugin } from './plugins/subdocument.plugin.js';
+
+// Utilities
+export {
+  getFieldsForUser,
+  getMongooseProjection,
+  filterResponseData,
+  createFieldPreset,
+} from './utils/field-selection.js';
+
+export * as actions from './actions/index.js';
+
+import { Repository } from './Repository.js';
+
+export const createRepository = (Model, plugins = []) => {
+  return new Repository(Model, plugins);
+};
+
+export default Repository;

package/src/pagination/PaginationEngine.js

@@ -0,0 +1,348 @@
+import { encodeCursor, decodeCursor, validateCursorSort, validateCursorVersion } from './utils/cursor.js';
+import { validateKeysetSort, getPrimaryField } from './utils/sort.js';
+import { buildKeysetFilter } from './utils/filter.js';
+import {
+  validateLimit,
+  validatePage,
+  shouldWarnDeepPagination,
+  calculateSkip,
+  calculateTotalPages
+} from './utils/limits.js';
+import { createError } from '../utils/error.js';
+
+/**
+ * @typedef {import('mongoose').Model} Model
+ * @typedef {import('mongoose').PopulateOptions} PopulateOptions
+ * @typedef {import('mongoose').ClientSession} ClientSession
+ */
+
+/**
+ * @typedef {Object} PaginationConfig
+ * @property {number} [defaultLimit=10] - Default number of documents per page
+ * @property {number} [maxLimit=100] - Maximum allowed limit
+ * @property {number} [maxPage=10000] - Maximum allowed page number
+ * @property {number} [deepPageThreshold=100] - Page number that triggers performance warning
+ * @property {number} [cursorVersion=1] - Cursor version for forward compatibility
+ * @property {boolean} [useEstimatedCount=false] - Use estimatedDocumentCount for faster counts on large collections
+ */
+
+/**
+ * @typedef {Object} OffsetPaginationOptions
+ * @property {Record<string, any>} [filters={}] - MongoDB query filters
+ * @property {Record<string, 1|-1>} [sort] - Sort specification
+ * @property {number} [page=1] - Page number (1-indexed)
+ * @property {number} [limit] - Number of documents per page
+ * @property {string|string[]} [select] - Fields to select
+ * @property {string|string[]|PopulateOptions|PopulateOptions[]} [populate] - Fields to populate
+ * @property {boolean} [lean=true] - Return plain JavaScript objects
+ * @property {ClientSession} [session] - MongoDB session for transactions
+ */
+
+/**
+ * @typedef {Object} KeysetPaginationOptions
+ * @property {Record<string, any>} [filters={}] - MongoDB query filters
+ * @property {Record<string, 1|-1>} [sort] - Sort specification (required at runtime)
+ * @property {string} [after] - Cursor token for next page
+ * @property {number} [limit] - Number of documents per page
+ * @property {string|string[]} [select] - Fields to select
+ * @property {string|string[]|PopulateOptions|PopulateOptions[]} [populate] - Fields to populate
+ * @property {boolean} [lean=true] - Return plain JavaScript objects
+ * @property {ClientSession} [session] - MongoDB session for transactions
+ */
+
+/**
+ * @typedef {Object} AggregatePaginationOptions
+ * @property {any[]} [pipeline=[]] - Aggregation pipeline stages
+ * @property {number} [page=1] - Page number (1-indexed)
+ * @property {number} [limit] - Number of documents per page
+ * @property {ClientSession} [session] - MongoDB session for transactions
+ */
+
+/**
+ * @typedef {Object} OffsetPaginationResult
+ * @property {'offset'} method - Pagination method used
+ * @property {any[]} docs - Array of documents
+ * @property {number} page - Current page number
+ * @property {number} limit - Documents per page
+ * @property {number} total - Total document count
+ * @property {number} pages - Total page count
+ * @property {boolean} hasNext - Whether next page exists
+ * @property {boolean} hasPrev - Whether previous page exists
+ * @property {string} [warning] - Performance warning for deep pagination
+ */
+
+/**
+ * @typedef {Object} KeysetPaginationResult
+ * @property {'keyset'} method - Pagination method used
+ * @property {any[]} docs - Array of documents
+ * @property {number} limit - Documents per page
+ * @property {boolean} hasMore - Whether more documents exist
+ * @property {string|null} next - Cursor token for next page
+ */
+
+/**
+ * @typedef {Object} AggregatePaginationResult
+ * @property {'aggregate'} method - Pagination method used
+ * @property {any[]} docs - Array of documents
+ * @property {number} page - Current page number
+ * @property {number} limit - Documents per page
+ * @property {number} total - Total document count
+ * @property {number} pages - Total page count
+ * @property {boolean} hasNext - Whether next page exists
+ * @property {boolean} hasPrev - Whether previous page exists
+ * @property {string} [warning] - Performance warning for deep pagination
+ */
+
+/**
+ * Production-grade pagination engine for MongoDB
+ * Supports offset, keyset (cursor), and aggregate pagination
+ *
+ * @example
+ * const engine = new PaginationEngine(UserModel, {
+ *   defaultLimit: 20,
+ *   maxLimit: 100,
+ *   useEstimatedCount: true
+ * });
+ *
+ * // Offset pagination
+ * const page1 = await engine.paginate({ page: 1, limit: 20 });
+ *
+ * // Keyset pagination (better for large datasets)
+ * const stream1 = await engine.stream({ sort: { createdAt: -1 }, limit: 20 });
+ * const stream2 = await engine.stream({ sort: { createdAt: -1 }, after: stream1.next });
+ */
+export class PaginationEngine {
+  /**
+   * Create a new pagination engine
+   *
+   * @param {Model} Model - Mongoose model to paginate
+   * @param {PaginationConfig} [config={}] - Pagination configuration
+   */
+  constructor(Model, config = {}) {
+    this.Model = Model;
+    this.config = {
+      defaultLimit: config.defaultLimit || 10,
+      maxLimit: config.maxLimit || 100,
+      maxPage: config.maxPage || 10000,
+      deepPageThreshold: config.deepPageThreshold || 100,
+      cursorVersion: config.cursorVersion || 1,
+      useEstimatedCount: config.useEstimatedCount || false
+    };
+  }
+
+  /**
+   * Offset-based pagination using skip/limit
+   * Best for small datasets and when users need random page access
+   * O(n) performance - slower for deep pages
+   *
+   * @param {OffsetPaginationOptions} [options={}] - Pagination options
+   * @returns {Promise<OffsetPaginationResult>} Pagination result with total count
+   *
+   * @example
+   * const result = await engine.paginate({
+   *   filters: { status: 'active' },
+   *   sort: { createdAt: -1 },
+   *   page: 1,
+   *   limit: 20
+   * });
+   * console.log(result.docs, result.total, result.hasNext);
+   */
+  async paginate(options = {}) {
+    const {
+      filters = {},
+      sort = { _id: -1 },
+      page = 1,
+      limit = this.config.defaultLimit,
+      select,
+      populate = [],
+      lean = true,
+      session
+    } = options;
+
+    const sanitizedPage = validatePage(page, this.config);
+    const sanitizedLimit = validateLimit(limit, this.config);
+    const skip = calculateSkip(sanitizedPage, sanitizedLimit);
+
+    let query = this.Model.find(filters);
+    if (select) query = query.select(select);
+    if (populate && (Array.isArray(populate) ? populate.length : populate)) query = query.populate(populate);
+    query = query.sort(sort).skip(skip).limit(sanitizedLimit).lean(lean);
+    if (session) query = query.session(session);
+
+    const hasFilters = Object.keys(filters).length > 0;
+    const useEstimated = this.config.useEstimatedCount && !hasFilters;
+
+    // Note: estimatedDocumentCount() doesn't support sessions or filters
+    // It reads collection metadata (O(1) instant), not actual documents
+    // Falls back to countDocuments() when filters are present
+    const [docs, total] = await Promise.all([
+      query,
+      useEstimated
+        ? this.Model.estimatedDocumentCount()
+        : this.Model.countDocuments(filters).session(session)
+    ]);
+
+    const totalPages = calculateTotalPages(total, sanitizedLimit);
+    const warning = shouldWarnDeepPagination(sanitizedPage, this.config.deepPageThreshold)
+      ? `Deep pagination (page ${sanitizedPage}). Consider getAll({ after, sort, limit }) for better performance.`
+      : undefined;
+
+    return /** @type {const} */ ({
+      method: 'offset',
+      docs,
+      page: sanitizedPage,
+      limit: sanitizedLimit,
+      total,
+      pages: totalPages,
+      hasNext: sanitizedPage < totalPages,
+      hasPrev: sanitizedPage > 1,
+      ...(warning && { warning })
+    });
+  }
+
+  /**
+   * Keyset (cursor-based) pagination for high-performance streaming
+   * Best for large datasets, infinite scroll, real-time feeds
+   * O(1) performance - consistent speed regardless of position
+   *
+   * @param {KeysetPaginationOptions} options - Pagination options (sort is required)
+   * @returns {Promise<KeysetPaginationResult>} Pagination result with next cursor
+   *
+   * @example
+   * // First page
+   * const page1 = await engine.stream({
+   *   sort: { createdAt: -1 },
+   *   limit: 20
+   * });
+   *
+   * // Next page using cursor
+   * const page2 = await engine.stream({
+   *   sort: { createdAt: -1 },
+   *   after: page1.next,
+   *   limit: 20
+   * });
+   */
+  async stream(options = {}) {
+    const {
+      filters = {},
+      sort,
+      after,
+      limit = this.config.defaultLimit,
+      select,
+      populate = [],
+      lean = true,
+      session
+    } = options;
+
+    if (!sort) {
+      throw createError(400, 'sort is required for keyset pagination');
+    }
+
+    const sanitizedLimit = validateLimit(limit, this.config);
+    const normalizedSort = validateKeysetSort(sort);
+
+    let query = { ...filters };
+
+    if (after) {
+      const cursor = decodeCursor(after);
+      validateCursorVersion(cursor.version, this.config.cursorVersion);
+      validateCursorSort(cursor.sort, normalizedSort);
+      query = buildKeysetFilter(query, normalizedSort, cursor.value, cursor.id);
+    }
+
+    let mongoQuery = this.Model.find(query);
+    if (select) mongoQuery = mongoQuery.select(select);
+    if (populate && (Array.isArray(populate) ? populate.length : populate)) mongoQuery = mongoQuery.populate(populate);
+    mongoQuery = mongoQuery.sort(normalizedSort).limit(sanitizedLimit + 1).lean(lean);
+    if (session) mongoQuery = mongoQuery.session(session);
+
+    const docs = await mongoQuery;
+
+    const hasMore = docs.length > sanitizedLimit;
+    if (hasMore) docs.pop();
+
+    const primaryField = getPrimaryField(normalizedSort);
+    const nextCursor = hasMore && docs.length > 0
+      ? encodeCursor(docs[docs.length - 1], primaryField, normalizedSort, this.config.cursorVersion)
+      : null;
+
+    return /** @type {const} */ ({
+      method: 'keyset',
+      docs,
+      limit: sanitizedLimit,
+      hasMore,
+      next: nextCursor
+    });
+  }
+
+  /**
+   * Aggregate pipeline with pagination
+   * Best for complex queries requiring aggregation stages
+   * Uses $facet to combine results and count in single query
+   *
+   * @param {AggregatePaginationOptions} [options={}] - Aggregation options
+   * @returns {Promise<AggregatePaginationResult>} Pagination result with total count
+   *
+   * @example
+   * const result = await engine.aggregatePaginate({
+   *   pipeline: [
+   *     { $match: { status: 'active' } },
+   *     { $group: { _id: '$category', count: { $sum: 1 } } },
+   *     { $sort: { count: -1 } }
+   *   ],
+   *   page: 1,
+   *   limit: 20
+   * });
+   */
+  async aggregatePaginate(options = {}) {
+    const {
+      pipeline = [],
+      page = 1,
+      limit = this.config.defaultLimit,
+      session
+    } = options;
+
+    const sanitizedPage = validatePage(page, this.config);
+    const sanitizedLimit = validateLimit(limit, this.config);
+    const skip = calculateSkip(sanitizedPage, sanitizedLimit);
+
+    const facetPipeline = [
+      ...pipeline,
+      {
+        $facet: {
+          docs: [
+            { $skip: skip },
+            { $limit: sanitizedLimit }
+          ],
+          total: [
+            { $count: 'count' }
+          ]
+        }
+      }
+    ];
+
+    const aggregation = this.Model.aggregate(facetPipeline);
+    if (session) aggregation.session(session);
+
+    const [result] = await aggregation.exec();
+    const docs = result.docs;
+    const total = result.total[0]?.count || 0;
+    const totalPages = calculateTotalPages(total, sanitizedLimit);
+
+    const warning = shouldWarnDeepPagination(sanitizedPage, this.config.deepPageThreshold)
+      ? `Deep pagination in aggregate (page ${sanitizedPage}). Uses $skip internally.`
+      : undefined;
+
+    return /** @type {const} */ ({
+      method: 'aggregate',
+      docs,
+      page: sanitizedPage,
+      limit: sanitizedLimit,
+      total,
+      pages: totalPages,
+      hasNext: sanitizedPage < totalPages,
+      hasPrev: sanitizedPage > 1,
+      ...(warning && { warning })
+    });
+  }
+}

package/src/pagination/utils/cursor.js

@@ -0,0 +1,119 @@
+import mongoose from 'mongoose';
+
+/**
+ * Encodes document values and sort metadata into a base64 cursor token
+ *
+ * @param {any} doc - Document to extract cursor values from
+ * @param {string} primaryField - Primary sort field name
+ * @param {Record<string, 1|-1>} sort - Normalized sort specification
+ * @param {number} [version=1] - Cursor version for forward compatibility
+ * @returns {string} Base64-encoded cursor token
+ */
+export function encodeCursor(doc, primaryField, sort, version = 1) {
+  const primaryValue = doc[primaryField];
+  const idValue = doc._id;
+
+  const payload = {
+    v: serializeValue(primaryValue),
+    t: getValueType(primaryValue),
+    id: serializeValue(idValue),
+    idType: getValueType(idValue),
+    sort,
+    ver: version
+  };
+
+  return Buffer.from(JSON.stringify(payload)).toString('base64');
+}
+
+/**
+ * Decodes a cursor token back into document values and sort metadata
+ *
+ * @param {string} token - Base64-encoded cursor token
+ * @returns {{value: any, id: any, sort: Record<string, 1|-1>, version: number}} Decoded cursor data
+ * @throws {Error} If token is invalid or malformed
+ */
+export function decodeCursor(token) {
+  try {
+    const json = Buffer.from(token, 'base64').toString('utf-8');
+    const payload = JSON.parse(json);
+
+    return {
+      value: rehydrateValue(payload.v, payload.t),
+      id: rehydrateValue(payload.id, payload.idType),
+      sort: payload.sort,
+      version: payload.ver
+    };
+  } catch (err) {
+    throw new Error('Invalid cursor token');
+  }
+}
+
+/**
+ * Validates that cursor sort matches current query sort
+ *
+ * @param {Record<string, 1|-1>} cursorSort - Sort specification from cursor
+ * @param {Record<string, 1|-1>} currentSort - Sort specification from query
+ * @throws {Error} If sorts don't match
+ */
+export function validateCursorSort(cursorSort, currentSort) {
+  const cursorSortStr = JSON.stringify(cursorSort);
+  const currentSortStr = JSON.stringify(currentSort);
+
+  if (cursorSortStr !== currentSortStr) {
+    throw new Error('Cursor sort does not match current query sort');
+  }
+}
+
+/**
+ * Validates cursor version matches expected version
+ *
+ * @param {number} cursorVersion - Version from cursor
+ * @param {number} expectedVersion - Expected version from config
+ * @throws {Error} If versions don't match
+ */
+export function validateCursorVersion(cursorVersion, expectedVersion) {
+  if (cursorVersion !== expectedVersion) {
+    throw new Error(`Cursor version ${cursorVersion} does not match expected version ${expectedVersion}`);
+  }
+}
+
+/**
+ * Serializes a value for cursor storage
+ * @param {any} value - Value to serialize
+ * @returns {string|number|boolean} Serialized value
+ */
+function serializeValue(value) {
+  if (value instanceof Date) return value.toISOString();
+  if (value instanceof mongoose.Types.ObjectId) return value.toString();
+  return value;
+}
+
+/**
+ * Gets the type identifier for a value
+ * @param {any} value - Value to identify
+ * @returns {'date'|'objectid'|'boolean'|'number'|'string'|'unknown'} Type identifier
+ */
+function getValueType(value) {
+  if (value instanceof Date) return 'date';
+  if (value instanceof mongoose.Types.ObjectId) return 'objectid';
+  if (typeof value === 'boolean') return 'boolean';
+  if (typeof value === 'number') return 'number';
+  if (typeof value === 'string') return 'string';
+  return 'unknown';
+}
+
+/**
+ * Rehydrates a serialized value back to its original type
+ * @param {any} serialized - Serialized value
+ * @param {string} type - Type identifier
+ * @returns {any} Rehydrated value
+ */
+function rehydrateValue(serialized, type) {
+  switch (type) {
+    case 'date': return new Date(serialized);
+    case 'objectid': return new mongoose.Types.ObjectId(serialized);
+    case 'boolean': return Boolean(serialized);
+    case 'number': return Number(serialized);
+    default: return serialized;
+  }
+}

package/src/pagination/utils/filter.js

@@ -0,0 +1,42 @@
+/**
+ * Builds MongoDB filter for keyset pagination
+ * Creates compound $or condition for proper cursor-based filtering
+ *
+ * @param {Record<string, any>} baseFilters - Existing query filters
+ * @param {Record<string, 1|-1>} sort - Normalized sort specification
+ * @param {any} cursorValue - Primary field value from cursor
+ * @param {any} cursorId - _id value from cursor
+ * @returns {Record<string, any>} MongoDB filter with keyset condition
+ *
+ * @example
+ * buildKeysetFilter(
+ *   { status: 'active' },
+ *   { createdAt: -1, _id: -1 },
+ *   new Date('2024-01-01'),
+ *   new ObjectId('...')
+ * )
+ * // Returns:
+ * // {
+ * //   status: 'active',
+ * //   $or: [
+ * //     { createdAt: { $lt: Date('2024-01-01') } },
+ * //     { createdAt: Date('2024-01-01'), _id: { $lt: ObjectId('...') } }
+ * //   ]
+ * // }
+ */
+export function buildKeysetFilter(baseFilters, sort, cursorValue, cursorId) {
+  const primaryField = Object.keys(sort).find(k => k !== '_id') || '_id';
+  const direction = sort[primaryField];
+  const operator = direction === 1 ? '$gt' : '$lt';
+
+  return {
+    ...baseFilters,
+    $or: [
+      { [primaryField]: { [operator]: cursorValue } },
+      {
+        [primaryField]: cursorValue,
+        _id: { [operator]: cursorId }
+      }
+    ]
+  };
+}