@byted-apaas/server-sdk-node 1.1.16 → 1.1.17-beta.11

package/context/db/impl/IObject.d.ts

@@ -155,14 +155,15 @@ export interface _IKQuery<T> {
      *   如果未设置排序字段，默认以 _id 增序查询；
      *   如果有设置排序字段，必须设置具有唯一属性的字段，否则会有数据重复的风险；
      * @param handler 业务处理函数
+     * @param pageLimit 分页查询的数量，可选参数，默认值为 200，
      * @example
      * ```
      * await application.data.object('_user').findStream(async (records) => {
      *     // doSomething ...
-     * });
+     * }, 300);
      * ```
      */
-    findStream(handler: (records: object[]) => Promise<void>): Promise<void>;
+    findStream: (handler: (records: object[]) => Promise<void>, pageLimit?: number) => Promise<void>;
     /**
      * 无需入参，返回符合条件的记录，单次返回 200 条
      * @example

package/context/db/impl/object.d.ts

@@ -1,7 +1,7 @@
 import { _Cond, _Record, _WhereCond } from '../../../types/types';
 import { _IKAsyncEndpoint, _IKQuery, _IKSyncEndpoint } from './IObject';
 import { AppCtx } from '../../../application/application';
-import { BatchResult } from "../../../common/structs";
+import { BatchResult } from '../../../common/structs';
 /**
  * _KObject is kunlun object, every method new a _KQuery object to deal.
  */
@@ -92,7 +92,7 @@ declare class _KQuery<T> implements _IKQuery<T> {
      * @Deprecated Use FindStream instead.
      */
     findAll(): Promise<_Record<T>[]>;
-    findStream(handler: (records: object[]) => Promise<void>): Promise<void>;
+    findStream(handler: (records: object[]) => Promise<void>, pageLimit?: number): Promise<void>;
     private walkRecordsByID;
     private walkRecordsByLimitOffset;
     private getRecordsByPage;

package/context/db/impl/object.js

@@ -139,9 +139,8 @@ class _KObjectSync {
         }
         recordMapList = permissionUtils.batchDelUnauthField(recordMapList).newRecords;
         let data = (this.appCtx && this.appCtx.mode == 'openSDK') ?
-            await (0, common_1.runCtxForOpenSDK)(this.appCtx, async () => {
-                return await Request.GetInstance().openSDKCreateRecordsBySync(this.apiName, recordMapList);
-            }) : await Request.GetInstance().createRecordsBySync(this.apiName, recordMapList, this.authType);
+            await (0, common_1.runCtxForOpenSDK)(this.appCtx, async () => await Request.GetInstance().openSDKCreateRecordsBySync(this.apiName, recordMapList))
+            : await Request.GetInstance().createRecordsBySync(this.apiName, recordMapList, this.authType);
         // todo: 确定函数返回值
         if (!(data instanceof Array) && data.record_ids) {
             return data.record_ids;
@@ -411,8 +410,11 @@ class _KQuery {
             });
         }
         else {
-            const criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName);
-            return await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), queryBuilder.fuzzySearch, queryBuilder.getOrder(), false, queryBuilder.getSelect(), queryBuilder.getOffset(), queryBuilder.getLimit(), false, this.authType);
+            let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName);
+            if (queryBuilder.getLogic().logics.length > 0) {
+                criterion = await (0, logic_1.handleCriterion)(criterion);
+            }
+            return await Request.GetInstance().getRecordsOrCountByCriterion(apiName, criterion, queryBuilder.fuzzySearch, queryBuilder.getOrder(), false, queryBuilder.getSelect(), queryBuilder.getOffset(), queryBuilder.getLimit(), false, this.authType);
         }
     }
     ;
@@ -437,7 +439,7 @@ class _KQuery {
         return await this.getRecordsByPage();
     }
     ;
-    async findStream(handler) {
+    async findStream(handler, pageLimit = 200) {
         const { queryBuilder, queryV2 } = queryPropertiesStore.get(this);
         let offset = 0, orders = [];
         if (queryV2) {
@@ -448,14 +450,17 @@ class _KQuery {
             offset = queryBuilder.getOffset();
             orders = queryBuilder.getOrder();
         }
+        if (pageLimit === 0) {
+            pageLimit = queryBuilder_1.defaultLimit;
+        }
         if (orders.length === 0 && offset === 0) {
             // 走主键翻页
-            return await this.walkRecordsByID(handler);
+            return await this.walkRecordsByID(handler, pageLimit);
         }
         // 走 limit offset 翻页
-        return await this.walkRecordsByLimitOffset(handler);
+        return await this.walkRecordsByLimitOffset(handler, pageLimit);
     }
-    async walkRecordsByID(handler) {
+    async walkRecordsByID(handler, pageLimit) {
         const { apiName, appCtx, queryBuilder, queryV2 } = queryPropertiesStore.get(this);
         let limit = 0, selectFields = [];
         if (queryV2) {
@@ -471,9 +476,23 @@ class _KQuery {
             selectFields = queryBuilder.getSelect();
         }
         let maxId = 0, queryCount = 0;
+        let criterionV2;
+        let criterion;
+        if (appCtx) {
+            criterionV2 = (0, logicV2_1.buildCriterionV2)(queryV2.filter);
+            criterionV2.push({ leftValue: '_id', operator: operator_1.operates.GT, rightValue: maxId });
+        }
+        else {
+            criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName);
+            if (queryBuilder.getLogic().logics.length > 0) { // 有自定义搜索条件时，需要获取 fields
+                criterion = await (0, logic_1.handleCriterion)(criterion);
+            }
+            // 在最后，添加 _id>maxId 条件
+            criterion = (0, logic_1.addIDCriterion)(criterion, apiName, maxId);
+        }
         while (true) {
-            let newLimit = queryBuilder_1.defaultLimit;
-            if (limit > 0 && limit - queryCount < queryBuilder_1.defaultLimit) {
+            let newLimit = pageLimit;
+            if (limit > 0 && limit - queryCount < pageLimit) {
                 newLimit = limit - queryCount;
             }
             if (newLimit <= 0) {
@@ -481,23 +500,24 @@ class _KQuery {
             }
             let rs;
             if (appCtx) {
-                let criterion = (0, logicV2_1.buildCriterionV2)(queryV2.filter);
-                criterion.push({ leftValue: '_id', operator: operator_1.operates.GT, rightValue: maxId });
+                criterionV2[criterionV2.length - 1].rightValue = maxId;
                 const param = {
                     limit: newLimit,
                     offset: 0,
                     sort: [{ field: '_id', direction: 'asc', type: '' }],
                     fields: selectFields,
                     count: false,
-                    filter: criterion,
+                    filter: criterionV2,
                 };
                 rs = await (0, common_1.runCtxForOpenSDK)(appCtx, async () => {
                     return await Request.GetInstance().openSDKGetRecords(apiName, param);
                 });
             }
             else {
-                let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName, maxId);
-                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), queryBuilder.fuzzySearch, [new order_1.Order('_id', 'asc')], false, queryBuilder.getSelect(), 0, newLimit, false, this.authType);
+                // 更新 maxId
+                criterion.conditions[criterion.conditions.length - 1].right.settings.data = maxId;
+                // 发起请求
+                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, criterion, queryBuilder.fuzzySearch, [new order_1.Order('_id', 'asc')], false, queryBuilder.getSelect(), 0, newLimit, false, this.authType);
             }
             queryCount += rs.length;
             rs.forEach((r) => {
@@ -511,12 +531,12 @@ class _KQuery {
             catch (err) {
                 throw err;
             }
-            if (rs.length < queryBuilder_1.defaultLimit) {
+            if (rs.length < pageLimit) {
                 break;
             }
         }
     }
-    async walkRecordsByLimitOffset(handler) {
+    async walkRecordsByLimitOffset(handler, pageLimit) {
         const { apiName, appCtx, queryBuilder, queryV2 } = queryPropertiesStore.get(this);
         let limit = 0, offset = 0, selectFields = [], orders = [];
         if (queryV2) {
@@ -540,38 +560,54 @@ class _KQuery {
             orders = [{ field: '_id', direction: 'asc', type: '' }];
         }
         let maxId = 0, queryCount = 0;
-        for (let i = offset; !limit || i < offset + limit; i += queryBuilder_1.defaultLimit) {
+        let criterionV2;
+        let criterion;
+        if (appCtx) {
+            criterionV2 = (0, logicV2_1.buildCriterionV2)(queryV2.filter);
+            criterionV2.push({ leftValue: '_id', operator: operator_1.operates.GT, rightValue: maxId });
+        }
+        else {
+            criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName);
+            if (queryBuilder.getLogic().logics.length > 0) { // 有自定义搜索条件时，需要获取 fields
+                criterion = await (0, logic_1.handleCriterion)(criterion);
+            }
+            // 在最后，添加 _id>maxId 条件
+            criterion = (0, logic_1.addIDCriterion)(criterion, apiName, maxId);
+        }
+        let curOffset = offset;
+        while (true) {
             let newLimit = function () {
                 if (limit == 0) {
-                    return queryBuilder_1.defaultLimit;
+                    return pageLimit;
                 }
-                if (offset + limit - i < queryBuilder_1.defaultLimit) {
-                    return offset + limit - i;
+                if (offset + limit - curOffset < pageLimit) {
+                    return offset + limit - curOffset;
                 }
-                return queryBuilder_1.defaultLimit;
+                return pageLimit;
             }();
             if (newLimit <= 0) {
                 break;
             }
             let rs;
             if (appCtx) {
-                let criterion = (0, logicV2_1.buildCriterionV2)(queryV2.filter);
-                criterion.push({ leftValue: '_id', operator: operator_1.operates.GT, rightValue: maxId });
+                criterionV2[criterionV2.length - 1].rightValue = maxId;
                 const param = {
                     limit: newLimit,
-                    offset: i,
+                    offset: curOffset,
                     sort: orders,
                     fields: selectFields,
                     count: false,
-                    filter: criterion,
+                    filter: criterionV2,
                 };
                 rs = await (0, common_1.runCtxForOpenSDK)(appCtx, async () => {
                     return await Request.GetInstance().openSDKGetRecords(apiName, param);
                 });
             }
             else {
-                let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName, maxId);
-                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), queryBuilder.fuzzySearch, orders, false, selectFields, i, newLimit, false, this.authType);
+                // 更新 maxId
+                criterion.conditions[criterion.conditions.length - 1].right.settings.data = maxId;
+                // 发起请求
+                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, criterion, queryBuilder.fuzzySearch, orders, false, selectFields, curOffset, newLimit, false, this.authType);
             }
             queryCount += rs.length;
             try {
@@ -582,9 +618,10 @@ class _KQuery {
             catch (err) {
                 throw err;
             }
-            if (rs.length < queryBuilder_1.defaultLimit) {
+            if (rs.length < pageLimit) {
                 break;
             }
+            curOffset += pageLimit;
         }
     }
     async getRecordsByPage() {
@@ -608,8 +645,12 @@ class _KQuery {
                 });
             }
             else {
-                let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName, maxId);
-                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), null, // FindAll 不再迭代
+                let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName);
+                if (queryBuilder.getLogic().logics.length > 0) {
+                    criterion = await (0, logic_1.handleCriterion)(criterion);
+                }
+                criterion = (0, logic_1.addIDCriterion)(criterion, apiName, maxId);
+                rs = await Request.GetInstance().getRecordsOrCountByCriterion(apiName, criterion, null, // FindAll 不再迭代
                 [new order_1.Order('_id', 'asc')], false, queryBuilder.getSelect(), 0, queryBuilder_1.defaultLimit, false, this.authType);
             }
             rs.forEach((r) => {
@@ -742,7 +783,10 @@ class _KQuery {
         }
         else {
             let criterion = (0, logic_1.buildCriterion)(queryBuilder.getLogic(), apiName);
-            return await Request.GetInstance().getRecordsOrCountByCriterion(apiName, await (0, logic_1.handleCriterion)(criterion), queryBuilder.fuzzySearch, queryBuilder.getOrder(), false, queryBuilder.getSelect(), queryBuilder.getOffset(), queryBuilder.getLimit(), true, this.authType);
+            if (queryBuilder.getLogic().logics.length > 0) {
+                criterion = await (0, logic_1.handleCriterion)(criterion);
+            }
+            return await Request.GetInstance().getRecordsOrCountByCriterion(apiName, criterion, queryBuilder.fuzzySearch, queryBuilder.getOrder(), false, queryBuilder.getSelect(), queryBuilder.getOffset(), queryBuilder.getLimit(), true, this.authType);
         }
     }
     ;

package/kunlun/operator/impl/logic.d.ts

@@ -24,8 +24,16 @@ export type Criterion = {
  * @param objectApiName
  * @param maxId 添加 _id>maxId 条件，为 -1 表示不使用该参数
  */
-export declare function buildCriterion(logic: Logic, objectApiName: string, maxId?: number): Criterion | string;
+export declare function buildCriterion(logic: Logic, objectApiName: string): Criterion;
 export declare function buildExpression(objectApiName: string, left: string, right: Condition | UserDataType, index?: number): Expression;
+/**
+ * 在 criterion 中添加 _id>maxId 的判断条件
+ *
+ * @param criterion
+ * @param apiName
+ * @param maxId 添加 _id>maxId 条件
+ */
+export declare function addIDCriterion(criterion: Criterion, apiName: string, maxId: number): Criterion;
 export declare class Logic {
     value: string;
     expressions: {
@@ -47,4 +55,4 @@ export declare class Logic {
     addLogic(logic: Logic): void;
 }
 export declare function buildLogic(param: (Logic | object)[], logicValue?: string): Logic;
-export declare function handleCriterion(criterion: string | Criterion): Promise<string | Criterion>;
+export declare function handleCriterion(criterion: Criterion): Promise<Criterion>;

package/kunlun/operator/impl/logic.js

@@ -2,7 +2,7 @@
 // Copyright 2022 ByteDance Ltd. and/or its affiliates
 // SPDX-License-Identifier: MIT
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.handleCriterion = exports.buildLogic = exports.Logic = exports.buildExpression = exports.buildCriterion = exports.Condition = void 0;
+exports.handleCriterion = exports.buildLogic = exports.Logic = exports.addIDCriterion = exports.buildExpression = exports.buildCriterion = exports.Condition = void 0;
 const expression_1 = require("./expression");
 const operator_1 = require("./operator");
 const server_common_node_1 = require("@byted-apaas/server-common-node");
@@ -59,7 +59,7 @@ function isKunlunTypeOrSerializedObject(obj) {
  * @param objectApiName
  * @param maxId 添加 _id>maxId 条件，为 -1 表示不使用该参数
  */
-function buildCriterion(logic, objectApiName, maxId = -1) {
+function buildCriterion(logic, objectApiName) {
     let index = 0;
     let logics = [];
     let expressions = [];
@@ -117,18 +117,10 @@ function buildCriterion(logic, objectApiName, maxId = -1) {
     f(logic);
     // 如果未设置filter，默认添加: _isDeleted = false
     if (expressions.length < 1) {
-        if (maxId >= 0) {
-            return `{"conditions":[{"index":1,"left":{"type":"metadataVariable","settings":{"fieldPath":
-            [{"objectApiName":"${objectApiName}","fieldApiName":"_isDeleted"}]}},"operator":"equals","right":
-            {"type":"constant","settings":{"data":false}}},{"index":2,"left":{"type":"metadataVariable","settings":
-            {"fieldPath":[{"objectApiName":"${objectApiName}","fieldApiName":"_id"}]}},"operator":"greaterThan","right":
-            {"type":"constant","settings":{"data":${maxId}}}}],"logic":"1 and 2"}`;
-        }
-        else {
-            return `{"conditions":[{"index":1,"left":{"type":"metadataVariable","settings":{"fieldPath":
-            [{"objectApiName":"${objectApiName}","fieldApiName":"_isDeleted"}]}},"operator":"equals","right":
-            {"type":"constant","settings":{"data":false}}}],"logic":"1"}`;
-        }
+        logics = [];
+        expressions.push(buildExpression(objectApiName, '_isDeleted', false, expressions.length + 1));
+        logics.push('1');
+        return { conditions: expressions, logic: logics.join('') };
     }
     let isContainDeleted = false;
     for (let i = 0; i < expressions.length; i++) {
@@ -148,11 +140,6 @@ function buildCriterion(logic, objectApiName, maxId = -1) {
         logics.splice(0, 0, '(');
         logics.push(' and ', `${expressions.length}`, ')');
     }
-    if (maxId >= 0) {
-        expressions.push(buildExpression(objectApiName, '_id', new Condition('_id', maxId, operator_1.operates.GT), expressions.length + 1));
-        logics.splice(0, 0, '(');
-        logics.push(' and ', `${expressions.length}`, ')');
-    }
     return { conditions: expressions, logic: logics.join('') };
 }
 exports.buildCriterion = buildCriterion;
@@ -183,6 +170,22 @@ function buildExpression(objectApiName, left, right, index = 1) {
     return null;
 }
 exports.buildExpression = buildExpression;
+/**
+ * 在 criterion 中添加 _id>maxId 的判断条件
+ *
+ * @param criterion
+ * @param apiName
+ * @param maxId 添加 _id>maxId 条件
+ */
+function addIDCriterion(criterion, apiName, maxId) {
+    criterion.conditions.push(buildExpression(apiName, '_id', new Condition('_id', maxId, operator_1.operates.GT), criterion.conditions.length + 1));
+    const logics = criterion.logic.split('');
+    logics.splice(0, 0, '(');
+    logics.push(' and ', `${criterion.conditions.length}`, ')');
+    criterion.logic = logics.join('');
+    return criterion;
+}
+exports.addIDCriterion = addIDCriterion;
 // 逻辑操作：and 和 or
 class Logic {
     constructor(expressions = null, logicValue = 'and') {
@@ -339,9 +342,6 @@ class Filter {
     }
 }
 async function handleCriterion(criterion) {
-    if (typeof criterion === 'string') {
-        return criterion;
-    }
     if (!criterion || !criterion.conditions || !(criterion.conditions instanceof Array)) {
         return criterion;
     }
@@ -355,7 +355,7 @@ async function handleCriterion(criterion) {
         }
         await handleCondition(condition, fieldRes);
     }
-    return JSON.stringify(criterion);
+    return criterion;
 }
 exports.handleCriterion = handleCriterion;
 async function handleLookupCondition(conditions) {
@@ -376,6 +376,7 @@ async function handleLookupCondition(conditions) {
     let lookupMap = new Map(), dateSet = new Set();
     let firstObjApiName = conditions[0].left.settings.fieldPath[0].objectApiName;
     let fields = [];
+    // 获取对象的字段
     if (queryMap.size === 1) {
         fields.push(await Request.GetInstance().getField(firstObjApiName, fieldName));
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@byted-apaas/server-sdk-node",
-  "version": "1.1.16",
+  "version": "1.1.17-beta.11",
   "description": "aPaaS Server SDK",
   "author": "zhouwexin <zhouwexin@bytedance.com>",
   "homepage": "",

package/request/openapi.js

@@ -296,9 +296,12 @@ async function getRecordsOrCountByCriterion(objectApiName, criterion, fuzzySearc
     if (needCount) {
         fieldApiNames = ['_id'];
     }
+    if (typeof criterion === 'string') {
+        criterion = JSON.parse(criterion);
+    }
     // 3.请求
     options.json = {
-        'criterion': JSON.parse(criterion),
+        criterion,
         order,
         'ignore_back_lookup_field': ignoreBackLookupField,
         'field_api_names': fieldApiNames,
@@ -646,10 +649,17 @@ async function oql(oql, args, namedArgs, authType) {
         }
         options.headers[constants_3.AuthTypeHttpHeader] = authType;
     }
-    let result = await openapi.doRequest(null, urlPath, options);
-    if (result && result.rows) {
-        permissionUtils.appendUnauthFieldRecordList("", result.rows, result.unauth_field_slice, true);
-        return result.rows;
+    try {
+        let result = await openapi.doRequest(null, urlPath, options);
+        if (result && result.rows) {
+            permissionUtils.appendUnauthFieldRecordList("", result.rows, result.unauth_field_slice, true);
+            return result.rows;
+        }
+    }
+    catch (e) {
+        if (e.msg.includes("TimeoutError")) {
+            throw new exceptions.CallOpenapiError(`OQL timeout, please check the query statement and the execution time of the query statement.`);
+        }
     }
     return [];
 }