@nocobase/database 1.7.0-beta.32 → 1.7.0-beta.34

package/lib/cursor-builder.d.ts

@@ -0,0 +1,38 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Sequelize } from 'sequelize';
+import { FindOptions } from './repository';
+import { Model } from './model';
+import { Collection } from './collection';
+export declare class SmartCursorBuilder {
+    private sequelize;
+    private tableName;
+    private collection;
+    constructor(sequelize: Sequelize, tableName: string, collection: Collection);
+    /**
+     * 根据表结构自动选择最优游标策略
+     */
+    private getBestCursorStrategy;
+    /**
+     * Cursor-based pagination query function.
+     * Ideal for large datasets (e.g., millions of rows)
+     * Note:
+     *  1. does not support jumping to arbitrary pages (e.g., "Page 5")
+     *  2. Requires a stable, indexed sort field (e.g. ID, createdAt)
+     *  3. If custom orderBy is used, it must match the cursor field(s) and direction, otherwise results may be incorrect or unstable.
+     * @param options
+     */
+    chunk(options: FindOptions & {
+        chunkSize: number;
+        callback: (rows: Model[], options: FindOptions) => Promise<void>;
+        find: (options: FindOptions) => Promise<any[]>;
+        beforeFind?: (options: FindOptions) => Promise<void>;
+        afterFind?: (rows: Model[], options: FindOptions) => Promise<void>;
+    }): Promise<void>;
+}

package/lib/cursor-builder.js

@@ -0,0 +1,307 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var cursor_builder_exports = {};
+__export(cursor_builder_exports, {
+  SmartCursorBuilder: () => SmartCursorBuilder
+});
+module.exports = __toCommonJS(cursor_builder_exports);
+var import_sequelize = require("sequelize");
+var import_lodash = __toESM(require("lodash"));
+const _SmartCursorBuilder = class _SmartCursorBuilder {
+  sequelize;
+  tableName;
+  collection;
+  constructor(sequelize, tableName, collection) {
+    this.sequelize = sequelize;
+    this.tableName = tableName;
+    this.collection = collection;
+  }
+  /**
+   * 根据表结构自动选择最优游标策略
+   */
+  async getBestCursorStrategy() {
+    let indexInfoSql = "";
+    const dialect = this.sequelize.getDialect();
+    if (dialect === "postgres") {
+      indexInfoSql = `
+      SELECT 
+        t.relname AS table_name,
+        i.relname AS index_name,
+        a.attname AS column_name,
+        array_position(ix.indkey, a.attnum) + 1 AS seq_in_index,
+        CASE 
+          WHEN ix.indisprimary THEN 1
+          WHEN ix.indisunique THEN 2
+          ELSE 3
+        END AS index_type,
+        -- \u5224\u65AD\u7D22\u5F15\u6392\u5E8F\u65B9\u5411 (0=ASC, 1=DESC)
+        CASE WHEN (ix.indoption[array_position(ix.indkey, a.attnum) - 1] & 1) = 1 
+             THEN 'DESC' ELSE 'ASC' 
+        END AS direction
+      FROM 
+        pg_class t,
+        pg_class i,
+        pg_index ix,
+        pg_attribute a,
+        pg_namespace n
+      WHERE 
+        t.oid = ix.indrelid
+        AND i.oid = ix.indexrelid
+        AND a.attrelid = t.oid
+        AND t.relnamespace = n.oid
+        AND a.attnum = ANY(ix.indkey)
+        AND t.relkind = 'r'
+        AND n.nspname = current_schema()
+        AND t.relname = $1
+      ORDER BY 
+        i.relname, 
+        array_position(ix.indkey, a.attnum)
+    `;
+    } else if (dialect === "mariadb" || dialect === "mysql") {
+      indexInfoSql = `
+      SELECT 
+        i.TABLE_NAME, 
+        i.INDEX_NAME, 
+        i.COLUMN_NAME,
+        i.SEQ_IN_INDEX,
+        CASE 
+          WHEN i.INDEX_NAME = 'PRIMARY' THEN 1
+          WHEN i.NON_UNIQUE = 0 THEN 2
+          ELSE 3 
+        END as INDEX_TYPE
+      FROM 
+        information_schema.STATISTICS i
+      WHERE 
+        i.TABLE_SCHEMA = DATABASE() 
+        AND i.TABLE_NAME = ?
+      ORDER BY 
+        i.INDEX_NAME, 
+        i.SEQ_IN_INDEX;
+    `;
+    }
+    const indexRows = await this.sequelize.query(indexInfoSql, {
+      type: import_sequelize.QueryTypes.SELECT,
+      replacements: [this.tableName],
+      raw: true
+    });
+    const indexes = /* @__PURE__ */ new Map();
+    const indexDirections = /* @__PURE__ */ new Map();
+    if (!indexRows || indexRows.length === 0) {
+      if (Array.isArray(this.collection.filterTargetKey)) {
+        return new CompositeKeyCursorStrategy(this.collection.filterTargetKey);
+      }
+      return new SingleColumnCursorStrategy(this.collection.filterTargetKey);
+    }
+    for (const row of indexRows) {
+      const indexName = dialect === "postgres" ? row.index_name : row.INDEX_NAME;
+      const columnName = dialect === "postgres" ? row.column_name : row.COLUMN_NAME;
+      const indexType = dialect === "postgres" ? row.index_type : row.INDEX_TYPE;
+      if (dialect === "postgres" && row.direction) {
+        if (!indexDirections.has(indexName)) {
+          indexDirections.set(indexName, /* @__PURE__ */ new Map());
+        }
+        indexDirections.get(indexName).set(columnName, row.direction);
+      }
+      if (!indexes.has(indexName)) {
+        indexes.set(indexName, {
+          name: indexName,
+          columns: [],
+          isPrimary: dialect === "postgres" ? indexType === 1 : indexName === "PRIMARY",
+          isUnique: indexType < 3
+        });
+      }
+      const index = indexes.get(row.INDEX_NAME);
+      index.columns[row.SEQ_IN_INDEX - 1] = row.COLUMN_NAME;
+    }
+    for (const index of indexes.values()) {
+      if (index.isPrimary) {
+        if (index.columns.length === 1) {
+          return new SingleColumnCursorStrategy(index.columns[0]);
+        } else {
+          if (dialect === "postgres" && indexDirections.has(index.name)) {
+            const directions = index.columns.map((col) => indexDirections.get(index.name).get(col) || "ASC");
+            return new CompositeKeyCursorStrategy(index.columns, directions);
+          } else {
+            return new CompositeKeyCursorStrategy(index.columns);
+          }
+        }
+      }
+    }
+    let singleColumnUniqueIndex = null;
+    let multiColumnUniqueIndex = null;
+    for (const index of indexes.values()) {
+      if (index.isUnique && !index.isPrimary) {
+        if (index.columns.length === 1 && !singleColumnUniqueIndex) {
+          singleColumnUniqueIndex = index;
+        } else if (index.columns.length > 1 && !multiColumnUniqueIndex) {
+          multiColumnUniqueIndex = index;
+        }
+      }
+    }
+    if (singleColumnUniqueIndex) {
+      return new SingleColumnCursorStrategy(singleColumnUniqueIndex.columns[0]);
+    }
+    if (multiColumnUniqueIndex) {
+      return new CompositeKeyCursorStrategy(multiColumnUniqueIndex.columns);
+    }
+    let anyIndex = null;
+    for (const index of indexes.values()) {
+      if (index.columns.length > 0 && !index.isPrimary && !index.isUnique) {
+        anyIndex = index;
+        break;
+      }
+    }
+    if (anyIndex) {
+      if (anyIndex.columns.length === 1) {
+        return new SingleColumnCursorStrategy(anyIndex.columns[0]);
+      } else {
+        return new CompositeKeyCursorStrategy(anyIndex.columns);
+      }
+    }
+  }
+  /**
+   * Cursor-based pagination query function.
+   * Ideal for large datasets (e.g., millions of rows)
+   * Note:
+   *  1. does not support jumping to arbitrary pages (e.g., "Page 5")
+   *  2. Requires a stable, indexed sort field (e.g. ID, createdAt)
+   *  3. If custom orderBy is used, it must match the cursor field(s) and direction, otherwise results may be incorrect or unstable.
+   * @param options
+   */
+  async chunk(options) {
+    const cursorStrategy = await this.getBestCursorStrategy();
+    let cursorRecord = null;
+    let hasMoreData = true;
+    let isFirst = true;
+    options.order = cursorStrategy.buildSort();
+    options["parseSort"] = false;
+    while (hasMoreData) {
+      if (!isFirst) {
+        options.where = cursorStrategy.buildWhere(options.where, cursorRecord);
+      }
+      if (isFirst) {
+        isFirst = false;
+      }
+      options.limit = options.chunkSize || 1e3;
+      if (options.beforeFind) {
+        await options.beforeFind(options);
+      }
+      const records = await options.find(import_lodash.default.omit(options, "callback", "beforeFind", "afterFind", "chunkSize", "find"));
+      if (options.afterFind) {
+        await options.afterFind(records, options);
+      }
+      if (records.length === 0) {
+        hasMoreData = false;
+        continue;
+      }
+      await options.callback(records, options);
+      cursorRecord = records[records.length - 1];
+    }
+  }
+};
+__name(_SmartCursorBuilder, "SmartCursorBuilder");
+let SmartCursorBuilder = _SmartCursorBuilder;
+const _SingleColumnCursorStrategy = class _SingleColumnCursorStrategy {
+  columnName;
+  constructor(columnName) {
+    this.columnName = columnName;
+  }
+  buildSort() {
+    return [[this.columnName, "ASC"]];
+  }
+  buildWhere(baseWhere, record) {
+    if (!record) {
+      return baseWhere;
+    }
+    return { ...baseWhere, [this.columnName]: { [import_sequelize.Op.gt]: record[this.columnName] } };
+  }
+};
+__name(_SingleColumnCursorStrategy, "SingleColumnCursorStrategy");
+let SingleColumnCursorStrategy = _SingleColumnCursorStrategy;
+const _CompositeKeyCursorStrategy = class _CompositeKeyCursorStrategy {
+  columns;
+  directions;
+  constructor(columns, directions) {
+    this.columns = columns;
+    this.directions = directions || Array(columns.length).fill("ASC");
+  }
+  buildSort() {
+    const orderBy = [];
+    for (let i = 0; i < this.columns.length; i++) {
+      orderBy.push([this.columns[i], this.directions[i]]);
+    }
+    return orderBy;
+  }
+  buildWhere(baseWhere, record) {
+    if (!record) {
+      return baseWhere;
+    }
+    const whereConditions = [];
+    for (let i = 0; i < this.columns.length; i++) {
+      const column = this.columns[i];
+      if (i > 0) {
+        const equalConditions = {};
+        for (let j = 0; j < i; j++) {
+          equalConditions[this.columns[j]] = record[this.columns[j]];
+        }
+        whereConditions.push({
+          ...equalConditions,
+          [column]: {
+            [import_sequelize.Op.gt]: record[column]
+          }
+        });
+      } else {
+        whereConditions.push({
+          [column]: {
+            [import_sequelize.Op.gt]: record[column]
+          }
+        });
+      }
+    }
+    const cursorCondition = {
+      [import_sequelize.Op.or]: whereConditions
+    };
+    return baseWhere ? { [import_sequelize.Op.and]: [baseWhere, cursorCondition] } : cursorCondition;
+  }
+};
+__name(_CompositeKeyCursorStrategy, "CompositeKeyCursorStrategy");
+let CompositeKeyCursorStrategy = _CompositeKeyCursorStrategy;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SmartCursorBuilder
+});

package/lib/fields/nanoid-field.js

@@ -51,10 +51,12 @@ const _NanoidField = class _NanoidField extends import_field.Field {
   bind() {
     super.bind();
     this.on("beforeValidate", this.listener);
+    this.on("beforeCreate", this.listener);
   }
   unbind() {
     super.unbind();
     this.off("beforeValidate", this.listener);
+    this.off("beforeCreate", this.listener);
   }
 };
 __name(_NanoidField, "NanoidField");

package/lib/fields/uuid-field.js

@@ -49,10 +49,12 @@ const _UuidField = class _UuidField extends import_field.Field {
   bind() {
     super.bind();
     this.on("beforeValidate", this.listener);
+    this.on("beforeCreate", this.listener);
   }
   unbind() {
     super.unbind();
     this.off("beforeValidate", this.listener);
+    this.off("beforeCreate", this.listener);
   }
 };
 __name(_UuidField, "UuidField");

package/lib/options-parser.d.ts

@@ -27,7 +27,9 @@ export declare class OptionsParser {
     isAssociation(key: string): boolean;
     isAssociationPath(path: string): boolean;
     filterByTkToWhereOption(): {};
-    toSequelizeParams(): any;
+    toSequelizeParams(options?: {
+        parseSort?: boolean;
+    }): any;
     /**
      * parser sort options
      * @param filterParams

package/lib/options-parser.js

@@ -111,7 +111,7 @@ const _OptionsParser = class _OptionsParser {
       [filterTargetKey]: filterByTkOption
     };
   }
-  toSequelizeParams() {
+  toSequelizeParams(options = { parseSort: true }) {
     var _a, _b;
     const queryParams = this.filterParser.toSequelizeParams();
     if ((_a = this.options) == null ? void 0 : _a.filterByTk) {
@@ -126,7 +126,8 @@ const _OptionsParser = class _OptionsParser {
       }
       queryParams.include.push(...import_lodash.default.castArray(this.options.include));
     }
-    return this.parseSort(this.parseFields(queryParams));
+    const fields = this.parseFields(queryParams);
+    return options.parseSort ? this.parseSort(fields) : fields;
   }
   /**
    * parser sort options

package/lib/repository.d.ts

@@ -20,6 +20,7 @@ import { HasManyRepository } from './relation-repository/hasmany-repository';
 import { HasOneRepository } from './relation-repository/hasone-repository';
 import { RelationRepository } from './relation-repository/relation-repository';
 import { valuesToFilter } from './utils/filter-utils';
+import { SmartCursorBuilder } from './cursor-builder';
 interface CreateManyOptions extends BulkCreateOptions {
     records: Values[];
 }
@@ -145,6 +146,7 @@ export declare class Repository<TModelAttributes extends {} = any, TCreationAttr
     database: Database;
     collection: Collection;
     model: ModelStatic<Model>;
+    cursorBuilder: SmartCursorBuilder;
     constructor(collection: Collection);
     static valuesToFilter: typeof valuesToFilter;
     /**
@@ -157,6 +159,25 @@ export declare class Repository<TModelAttributes extends {} = any, TCreationAttr
     chunk(options: FindOptions & {
         chunkSize: number;
         callback: (rows: Model[], options: FindOptions) => Promise<void>;
+        beforeFind?: (options: FindOptions) => Promise<void>;
+        afterFind?: (rows: Model[], options: FindOptions & {
+            offset: number;
+        }) => Promise<void>;
+    }): Promise<void>;
+    /**
+     * Cursor-based pagination query function.
+     * Ideal for large datasets (e.g., millions of rows)
+     * Note:
+     *  1. does not support jumping to arbitrary pages (e.g., "Page 5")
+     *  2. Requires a stable, indexed sort field (e.g. ID, createdAt)
+     *  3. If custom orderBy is used, it must match the cursor field(s) and direction, otherwise results may be incorrect or unstable.
+     * @param options
+     */
+    chunkWithCursor(options: FindOptions & {
+        chunkSize: number;
+        callback: (rows: Model[], options: FindOptions) => Promise<void>;
+        beforeFind?: (options: FindOptions) => Promise<void>;
+        afterFind?: (rows: Model[], options: FindOptions) => Promise<void>;
     }): Promise<void>;
     /**
      * find

package/lib/repository.js

@@ -70,6 +70,8 @@ var import_hasone_repository = require("./relation-repository/hasone-repository"
 var import_update_associations = require("./update-associations");
 var import_update_guard = require("./update-guard");
 var import_filter_utils = require("./utils/filter-utils");
+var import_lodash2 = __toESM(require("lodash"));
+var import_cursor_builder = require("./cursor-builder");
 var import_sequelize2 = require("sequelize");
 const debug = require("debug")("noco-database");
 const transaction = (0, import_transaction_decorator.transactionWrapperBuilder)(function() {
@@ -121,10 +123,12 @@ const _Repository = class _Repository {
   database;
   collection;
   model;
+  cursorBuilder;
   constructor(collection) {
     this.database = collection.context.database;
     this.collection = collection;
     this.model = collection.model;
+    this.cursorBuilder = new import_cursor_builder.SmartCursorBuilder(this.database.sequelize, this.model.tableName, this.collection);
   }
   /**
    * return count by filter
@@ -205,18 +209,25 @@ const _Repository = class _Repository {
     return await this.model.aggregate(field, method, queryOptions);
   }
   async chunk(options) {
-    const { chunkSize, callback, limit: overallLimit } = options;
+    const { chunkSize, callback, limit: overallLimit, beforeFind, afterFind } = options;
     const transaction2 = await this.getTransaction(options);
     let offset = 0;
     let totalProcessed = 0;
     while (true) {
       const currentLimit = overallLimit !== void 0 ? Math.min(chunkSize, overallLimit - totalProcessed) : chunkSize;
-      const rows = await this.find({
+      const findOptions = {
         ...options,
         limit: currentLimit,
         offset,
         transaction: transaction2
-      });
+      };
+      if (beforeFind) {
+        await beforeFind(findOptions);
+      }
+      const rows = await this.find(findOptions);
+      if (afterFind) {
+        await afterFind(rows, { ...findOptions, offset });
+      }
       if (rows.length === 0) {
         break;
       }
@@ -228,6 +239,21 @@ const _Repository = class _Repository {
       }
     }
   }
+  /**
+   * Cursor-based pagination query function.
+   * Ideal for large datasets (e.g., millions of rows)
+   * Note:
+   *  1. does not support jumping to arbitrary pages (e.g., "Page 5")
+   *  2. Requires a stable, indexed sort field (e.g. ID, createdAt)
+   *  3. If custom orderBy is used, it must match the cursor field(s) and direction, otherwise results may be incorrect or unstable.
+   * @param options
+   */
+  async chunkWithCursor(options) {
+    return await this.cursorBuilder.chunk({
+      ...options,
+      find: this.find.bind(this)
+    });
+  }
   /**
    * find
    * @param options
@@ -535,7 +561,7 @@ const _Repository = class _Repository {
     const parser = new import_options_parser.OptionsParser(options, {
       collection: this.collection
     });
-    const params = parser.toSequelizeParams();
+    const params = parser.toSequelizeParams({ parseSort: import_lodash2.default.isBoolean(options == null ? void 0 : options.parseSort) ? options.parseSort : true });
     debug("sequelize query params %o", params);
     if (options.where && params.where) {
       params.where = {

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@nocobase/database",
-  "version": "1.7.0-beta.32",
+  "version": "1.7.0-beta.34",
   "description": "",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
   "license": "AGPL-3.0",
   "dependencies": {
-    "@nocobase/logger": "1.7.0-beta.32",
-    "@nocobase/utils": "1.7.0-beta.32",
+    "@nocobase/logger": "1.7.0-beta.34",
+    "@nocobase/utils": "1.7.0-beta.34",
     "async-mutex": "^0.3.2",
     "chalk": "^4.1.1",
     "cron-parser": "4.4.0",
@@ -38,5 +38,5 @@
     "url": "git+https://github.com/nocobase/nocobase.git",
     "directory": "packages/database"
   },
-  "gitHead": "d78ca2740143209c6a16002e0c36e3256ed5fd88"
+  "gitHead": "2795864162d6151f0f8cf33945b0b8b8a4cb20dc"
 }