@byted-apaas/server-sdk-node 1.0.12 → 1.0.14

package/README.md

@@ -1 +1,11 @@
-## aPaaS Server SDK 版本说明：
+## aPaaS @byted-apaas/server-sdk-node 版本说明：
+
+-------
+### 版本：1.0.13｜兼容升级
+新功能：支持流式查询 findStream
+```js
+await context.db.object("_user").findStream(async (records) => {
+    // doSomething ...
+});
+```
+优化：findAll 标记为过时方法，用 findStream 替代

package/context/db/impl/IObject.d.ts

@@ -121,6 +121,8 @@ export interface _IKAsyncEndpoint<T> {
 }
 export interface _IKQuery<T> {
     /**
+     * @Deprecated Use FindStream instead.
+     *
      * 无需入参，返回全部符合条件的记录
      * @example
      * ```
@@ -130,6 +132,17 @@ export interface _IKQuery<T> {
      * ```
      */
     findAll(): Promise<_Record<T>[]>;
+    /**
+     * 遍历全部符合条件的记录
+     * @param handler 业务处理函数
+     * @example
+     * ```
+     * await context.db.object("_user").findStream(async (records) => {
+     *     // doSomething ...
+     * });
+     * ```
+     */
+    findStream(handler: (records: object[]) => Promise<void>): Promise<void>;
     /**
      * 无需入参，返回符合条件的记录，单次返回 200 条
      * @example

package/context/db/impl/object.d.ts

@@ -56,6 +56,7 @@ declare class _KObjectQuery<T> implements _IKQuery<T> {
     find(): Promise<_Record<T>[]>;
     findOne(): Promise<_Record<T>>;
     findAll(): Promise<_Record<T>[]>;
+    findStream(handler: (records: object[]) => Promise<void>): Promise<void>;
     orderBy<K extends keyof T>(fieldApiNames: K[]): _KQuery<T>;
     orderBy<K extends keyof T>(...fieldApiNames: K[]): _KQuery<T>;
     orderByDesc<K extends keyof T>(fieldApiNames: K[]): _KQuery<T>;
@@ -75,7 +76,13 @@ declare class _KQuery<T> implements _IKQuery<T> {
     constructor(objectApiName: string, appCtx?: AppCtx);
     find(): Promise<_Record<T>[]>;
     findOne(): Promise<_Record<T>>;
+    /**
+     * @Deprecated Use FindStream instead.
+     */
     findAll(): Promise<_Record<T>[]>;
+    findStream(handler: (records: object[]) => Promise<void>): Promise<void>;
+    private walkRecordsByID;
+    private walkRecordsByLimitOffset;
     private getRecordsByPage;
     orderBy<K extends keyof T>(fieldApiNames: K[]): this;
     orderBy<K extends keyof T>(...fieldApiNames: K[]): this;
@@ -88,5 +95,6 @@ declare class _KQuery<T> implements _IKQuery<T> {
     limit(limit: number): this;
     offset(offset: number): this;
     count(): Promise<number>;
+    private findPreCheck;
 }
 export {};

package/context/db/impl/object.js

@@ -301,6 +301,9 @@ class _KObjectQuery {
         return new _KQuery(this.apiName, this.appCtx).findAll();
     }
     ;
+    async findStream(handler) {
+        return new _KQuery(this.apiName, this.appCtx).findStream(handler);
+    }
     orderBy(fieldApiNames, ...restFieldApiNames) {
         fieldApiNames = !fieldApiNames ? [] : fieldApiNames;
         return new _KQuery(this.apiName, this.appCtx).orderBy(server_common_node_1.utils.argsToList(fieldApiNames, restFieldApiNames));
@@ -350,23 +353,8 @@ class _KQuery {
         });
     }
     async find() {
+        this.findPreCheck();
         const { apiName, appCtx, queryBuilder, queryV2 } = queryPropertiesStore.get(this);
-        // 查询非 _id 字段或者关联对象下钻._id, limit 不允许超过 200
-        let checkLimit;
-        let checkFields;
-        if (queryV2) {
-            checkLimit = queryV2._limit;
-            checkFields = queryV2.fields;
-        }
-        else {
-            checkLimit = queryBuilder.getLimit();
-            checkFields = queryBuilder.getSelect();
-        }
-        if (checkFields.length != 1 || !(checkFields[0] == "_id" || checkFields[0].endsWith("._id"))) {
-            if (checkLimit > 200) {
-                throw new Error(`limit(${checkLimit}) > ${queryBuilder_1.maxLimit}. Limit should be 1~${queryBuilder_1.maxLimitV2} when querying the id field, otherwise it should be 1~${queryBuilder_1.maxLimit}`);
-            }
-        }
         if (appCtx) {
             const param = {
                 limit: queryV2._limit,
@@ -393,6 +381,9 @@ class _KQuery {
         return records instanceof Array ? (records.length > 0 ? records[0] : null) : null;
     }
     ;
+    /**
+     * @Deprecated Use FindStream instead.
+     */
     async findAll() {
         const { queryBuilder, queryV2 } = queryPropertiesStore.get(this);
         if (queryBuilder.getOrder().length > 0) {
@@ -404,6 +395,152 @@ class _KQuery {
         return await this.getRecordsByPage();
     }
     ;
+    async findStream(handler) {
+        const { queryBuilder, queryV2 } = queryPropertiesStore.get(this);
+        let offset = 0, orders = [];
+        if (queryV2) {
+            offset = queryV2._offset;
+            orders = queryV2._order;
+        }
+        else {
+            offset = queryBuilder.getOffset();
+            orders = queryBuilder.getOrder();
+        }
+        if (orders.length === 0 && offset === 0) {
+            // 走主键翻页
+            return await this.walkRecordsByID(handler);
+        }
+        // 走 limit offset 翻页
+        return await this.walkRecordsByLimitOffset(handler);
+    }
+    async walkRecordsByID(handler) {
+        const { apiName, appCtx, queryBuilder, queryV2 } = queryPropertiesStore.get(this);
+        let limit = 0, selectFields = [];
+        if (queryV2) {
+            if (queryV2.isSetLimit) {
+                limit = queryV2._limit;
+            }
+            selectFields = queryV2.fields;
+        }
+        else {
+            if (queryBuilder.isSetLimit) {
+                limit = queryBuilder.getLimit();
+            }
+            selectFields = queryBuilder.getSelect();
+        }
+        let maxId = 0, queryCount = 0;
+        while (true) {
+            let newLimit = queryBuilder_1.defaultLimit;
+            if (limit > 0 && limit - queryCount < queryBuilder_1.defaultLimit) {
+                newLimit = limit - queryCount;
+            }
+            if (newLimit <= 0) {
+                break;
+            }
+            let rs;
+            if (appCtx) {
+                let criterion = (0, logicV2_1.buildCriterionV2)(queryV2.filter);
+                criterion.push({ leftValue: '_id', operator: operator_1.operates.GT, rightValue: maxId });
+                const param = {
+                    limit: newLimit,
+                    offset: 0,
+                    sort: [{ field: '_id', direction: 'asc', type: '' }],
+                    fields: selectFields,
+                    count: false,
+                    filter: criterion,
+                };
+                rs = await (0, common_1.runCtxForOpenSDK)(appCtx, async () => {
+                    return await Request.GetInstance().openSDKGetRecords(apiName, param);
+                });
+            }
+            else {
+                let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName, maxId);
+                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), [new order_1.Order("_id", "asc")], false, queryBuilder.getSelect(), 0, newLimit, false);
+            }
+            queryCount += rs.length;
+            rs.forEach((r) => {
+                maxId = r._id > maxId ? r._id : maxId;
+            });
+            try {
+                if (rs.length > 0) {
+                    await handler(rs);
+                }
+            }
+            catch (err) {
+                throw err;
+            }
+            if (rs.length < queryBuilder_1.defaultLimit) {
+                break;
+            }
+        }
+    }
+    async walkRecordsByLimitOffset(handler) {
+        const { apiName, appCtx, queryBuilder, queryV2 } = queryPropertiesStore.get(this);
+        let limit = 0, offset = 0, selectFields = [], orders = [];
+        if (queryV2) {
+            if (queryV2.isSetLimit) {
+                limit = queryV2._limit;
+            }
+            offset = queryV2._offset;
+            selectFields = queryV2.fields;
+            orders = queryV2._order;
+        }
+        else {
+            if (queryBuilder.isSetLimit) {
+                limit = queryBuilder.getLimit();
+            }
+            offset = queryBuilder.getOffset();
+            selectFields = queryBuilder.getSelect();
+            orders = queryBuilder.getOrder();
+        }
+        let maxId = 0, queryCount = 0;
+        for (let i = offset; !limit || i < offset + limit; i += queryBuilder_1.defaultLimit) {
+            let newLimit = function () {
+                if (limit == 0) {
+                    return queryBuilder_1.defaultLimit;
+                }
+                if (offset + limit - i < queryBuilder_1.defaultLimit) {
+                    return offset + limit - i;
+                }
+                return queryBuilder_1.defaultLimit;
+            }();
+            if (newLimit <= 0) {
+                break;
+            }
+            let rs;
+            if (appCtx) {
+                let criterion = (0, logicV2_1.buildCriterionV2)(queryV2.filter);
+                criterion.push({ leftValue: '_id', operator: operator_1.operates.GT, rightValue: maxId });
+                const param = {
+                    limit: newLimit,
+                    offset: i,
+                    sort: orders,
+                    fields: selectFields,
+                    count: false,
+                    filter: criterion,
+                };
+                rs = await (0, common_1.runCtxForOpenSDK)(appCtx, async () => {
+                    return await Request.GetInstance().openSDKGetRecords(apiName, param);
+                });
+            }
+            else {
+                let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName, maxId);
+                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), orders, false, selectFields, i, newLimit, false);
+            }
+            queryCount += rs.length;
+            try {
+                if (rs.length > 0) {
+                    await handler(rs);
+                }
+            }
+            catch (err) {
+                throw err;
+            }
+            if (rs.length < queryBuilder_1.defaultLimit) {
+                break;
+            }
+        }
+    }
     async getRecordsByPage() {
         const { apiName, appCtx, queryBuilder, queryV2 } = queryPropertiesStore.get(this);
         let maxId = 0, records = [];
@@ -426,13 +563,13 @@ class _KQuery {
             }
             else {
                 let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName, maxId);
-                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), [new order_1.Order("_id", "asc")], false, queryBuilder.getSelect(), 0, queryBuilder_1.maxLimit, false);
+                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), [new order_1.Order("_id", "asc")], false, queryBuilder.getSelect(), 0, queryBuilder_1.defaultLimit, false);
             }
             rs.forEach((r) => {
                 maxId = r._id > maxId ? r._id : maxId;
             });
             records.push(...rs);
-            if (rs.length < queryBuilder_1.maxLimit) {
+            if (rs.length < queryBuilder_1.defaultLimit) {
                 break;
             }
         }
@@ -542,6 +679,20 @@ class _KQuery {
         }
     }
     ;
+    findPreCheck() {
+        const { queryBuilder, queryV2 } = queryPropertiesStore.get(this);
+        let limit = queryBuilder_1.defaultLimit;
+        if (queryV2) {
+            limit = queryV2._limit;
+        }
+        else {
+            limit = queryBuilder.limit;
+        }
+        if (limit < 1) {
+            // need import exceptions
+            throw new server_common_node_1.exceptions.InvalidParamError(`limit(${limit}) < 1`);
+        }
+    }
 }
 function applyMixins(derivedCtor, constructors) {
     constructors.forEach((baseCtor) => {

package/context/db/impl/queryBuilder.d.ts

@@ -1,7 +1,6 @@
 import { Order, OrderType } from './order';
 import { Logic } from '../../../kunlun/operator/impl/logic';
-export declare const maxLimit = 200;
-export declare const maxLimitV2 = 10000;
+export declare const defaultLimit = 200;
 /**
  * 查询构造器，在 DB 中被初始化，用于构建一次查询
  */
@@ -11,6 +10,7 @@ export declare class QueryBuilder {
     select: string[];
     where: Logic[];
     limit: number;
+    isSetLimit: boolean;
     offset: number;
     order: Order[];
     logic: Logic;

package/context/db/impl/queryBuilder.js

@@ -2,13 +2,12 @@
 // Copyright 2022 ByteDance Ltd. and/or its affiliates
 // SPDX-License-Identifier: MIT
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.QueryBuilder = exports.maxLimitV2 = exports.maxLimit = void 0;
+exports.QueryBuilder = exports.defaultLimit = void 0;
 const order_1 = require("./order");
 const logic_1 = require("../../../kunlun/operator/impl/logic");
 const server_common_node_1 = require("@byted-apaas/server-common-node");
-exports.maxLimit = 200;
-// 限制开放至 1w，当仅查询 _id 时，最大可返回 1w，其他查询字段时仅返回 200，由底层控制。
-exports.maxLimitV2 = 10000;
+// SDK 仅作 limit 小于 0 校验，其他校验拦截由底层控制，如当查询结果不满足受控条件时，limit 仅支持最大 200
+exports.defaultLimit = 200;
 /**
  * 查询构造器，在 DB 中被初始化，用于构建一次查询
  */
@@ -18,7 +17,7 @@ class QueryBuilder {
         this.dataSource = '';
         this.select = [];
         this.where = null;
-        this.limit = exports.maxLimit;
+        this.limit = exports.defaultLimit;
         this.offset = 0;
         this.order = [];
         this.logic = new logic_1.Logic();
@@ -60,15 +59,8 @@ class QueryBuilder {
         this.offset = offset;
     }
     setLimit(limit) {
-        if (limit < 1) {
-            // need import exceptions
-            throw new server_common_node_1.exceptions.InvalidParamError(`limit(${limit}) < 1`);
-        }
-        if (limit > exports.maxLimitV2) {
-            // need import exceptions
-            throw new server_common_node_1.exceptions.InvalidParamError(`limit(${limit}) > ${exports.maxLimitV2}. Limit should be 1~${exports.maxLimitV2} when querying the id field, otherwise it should be 1~${exports.maxLimit}`);
-        }
         this.limit = limit;
+        this.isSetLimit = true;
     }
     getLimit() {
         return this.limit;

package/kunlun/operator/impl/logicV2.d.ts

@@ -36,6 +36,7 @@ export declare class LogicalExpression {
 export declare class QueryV2 {
     objectApiName: string;
     _limit: number;
+    isSetLimit: boolean;
     _offset: number;
     fields: string[];
     _order: OrderV2[];

package/kunlun/operator/impl/logicV2.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildCriterionV2 = exports.logical = exports.QueryV2 = exports.LogicalExpression = exports.ArithmeticExpression = exports.NewConstantExpressionField = exports.NewMetadataExpressionField = void 0;
 const operator_1 = require("./operator");
-const queryBuilder_1 = require("../../../context/db/impl/queryBuilder");
 function NewMetadataExpressionField(objectApiName, fieldApiName) {
     return {
         type: "metadataVariable",
@@ -110,10 +109,8 @@ class QueryV2 {
         return this;
     }
     limit(limit) {
-        if (limit <= 0 || limit > 10000) {
-            throw new Error(`limit(${limit}) > ${queryBuilder_1.maxLimitV2}. Limit should be 1~${queryBuilder_1.maxLimitV2} when querying the id field, otherwise it should be 1~${queryBuilder_1.maxLimit}`);
-        }
         this._limit = limit;
+        this.isSetLimit = true;
         return this;
     }
     orderBy(fieldApiNames) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@byted-apaas/server-sdk-node",
-  "version": "1.0.12",
+  "version": "1.0.14",
   "description": "aPaaS Server SDK",
   "author": "zhouwexin <zhouwexin@bytedance.com>",
   "homepage": "",