dolphindb 2.0.947 → 2.0.949

package/index.js

@@ -4,7 +4,7 @@ import DayjsCustomParseFormat from 'dayjs/plugin/customParseFormat.js';
 dayjs.extend(DayjsCustomParseFormat);
 import ipaddrjs from 'ipaddr.js';
 const { fromByteArray: buf2ipaddr } = ipaddrjs;
-import { concat, assert, inspect, typed_array_to_buffer, connect_websocket, WebSocket, defer } from 'xshell';
+import { concat, assert, inspect, typed_array_to_buffer, connect_websocket, WebSocket, Lock } from 'xshell';
 import { t } from './i18n/index.js';
 export var DdbForm;
 (function (DdbForm) {
@@ -119,7 +119,7 @@ export const nulls = {
     bytes16: Uint8Array.from(new Array(16).fill(0))
 };
 /** 可以表示所有 DolphinDB 数据库中的数据类型  Can represent data types in all DolphinDB databases */
-export class DdbObj {
+class DdbObj {
     static dec = new TextDecoder('utf-8');
     static enc = new TextEncoder();
     /** 维护已解析的 symbol base，比如流数据中后续的 symbol 向量可能只发送一个 base.id, base.size == 0, 依赖之前发送的 symbol base ？
@@ -1275,6 +1275,7 @@ export class DdbObj {
         return obj;
     }
 }
+export { DdbObj };
 /** 整数一定用这个 number formatter, InspectOptions.decimals 不传也用这个 */
 let default_formatter = Intl.NumberFormat('en-US', { maximumFractionDigits: 20 });
 let _decimals = 20;
@@ -2158,7 +2159,7 @@ export class DdbDatabaseError extends Error {
         this.options = options;
     }
 }
-export class DDB {
+class DDB {
     /** 当前的 session id (http 或 tcp) */
     sid = '0';
     /** utf-8 text decoder */
@@ -2173,7 +2174,8 @@ export class DDB {
         e.g. `ws://127.0.0.1:8848/`, `wss://dolphindb.com`
     */
     url;
-    websocket = null;
+    /** 为所有 websocket 操作加锁，包括设置 this.on_message, this.on_error, websocket.send */
+    lwebsocket = new Lock();
     /** little endian (server) */
     le = true;
     /** little endian (client) */
@@ -2203,15 +2205,15 @@ export class DDB {
     error;
     /** DdbMessage listeners */
     listeners = [];
-    /** 为所有 websocket 操作加锁，包括设置 this.on_message, this.on_error, websocket.send */
-    pwebsocket = defer(null);
-    /** 为定义 pnode_run 函数加锁，避免重复定义 */
-    ppnode_run = defer(null);
+    /** 首次 connect 连接的 promise, 后面的 connect 调用都返回这个 */
+    pconnect;
+    /** 首次定义 pnode_run 的 promise，保证并发调用 rpc 时只定义一次 pnode_run */
+    ppnode_run;
     get connected() {
-        return !this.error && this.websocket?.readyState === WebSocket.OPEN;
+        return !this.error && this.lwebsocket.resource?.readyState === WebSocket.OPEN;
     }
     get errored() {
-        return this.error || (this.websocket && this.websocket.readyState !== WebSocket.OPEN);
+        return Boolean(this.error || (this.lwebsocket.resource && this.lwebsocket.resource.readyState !== WebSocket.OPEN));
     }
     /**
         使用 WebSocket URL 初始化连接到 DolphinDB 的实例（不建立实际的网络连接）
@@ -2277,62 +2279,46 @@ export class DDB {
             throw this.error || new DdbConnectionError(this);
         if (this.connected)
             return;
-        // 准备进入 websocket 临界区
-        const ptail = this.pwebsocket;
-        let pwebsocket = this.pwebsocket = defer();
-        // 避免 pwebsocket.reject 后出现 triggerUncaughtException
-        pwebsocket.catch(() => { });
-        // 上一个请求出现了 websocket 错误，那么直接抛出异常，且通过 reject 当前锁 (pwebsocket)，安全退出临界区
-        try {
-            await ptail;
-            // 已进入 websocket 临界区
-        }
-        catch (error) {
-            pwebsocket.reject(error);
-            throw error;
-        }
-        if (this.errored) {
-            const error = this.error || new DdbConnectionError(this);
-            pwebsocket.reject(error);
-            throw error;
-        }
-        if (this.connected) {
-            pwebsocket.resolve();
-            return;
-        }
-        this.on_error = () => {
-            pwebsocket.reject(this.error /* 一定有，不需要再 || new DdbConnectionError(this)  */);
-        };
-        try {
-            this.websocket = await connect_websocket(this.url, {
-                protocols: (() => {
-                    if (this.streaming)
-                        return 'streaming';
-                    if (this.python)
-                        return 'python';
-                })(),
-                on_message: (buffer, websocket) => {
-                    this.on_message(buffer, websocket);
-                },
-                on_error: error => {
-                    this.error ??= new DdbConnectionError(this, error);
-                    this.on_error();
-                }
-            });
-        }
-        catch (error) {
-            this.error ??= new DdbConnectionError(this, error);
-            pwebsocket.reject(error);
-            throw this.error;
-        }
-        assert(this.connected);
-        pwebsocket.resolve();
-        // websocket 临界区结束
-        await this.rpc('connect', {});
-        if (this.autologin)
-            await this.call('login', [this.username, this.password], { urgent: true });
-        if (this.streaming)
-            await this.subscribe();
+        return this.pconnect ??= new Promise(async (resolve, reject) => {
+            this.on_error = () => {
+                reject(this.error /* 一定有，不需要再 || new DdbConnectionError(this)  */);
+            };
+            try {
+                // 连接建立之前应该不会有别的调用占用 this.lwebsocket
+                this.lwebsocket.resource = await connect_websocket(this.url, {
+                    protocols: (() => {
+                        if (this.streaming)
+                            return 'streaming';
+                        if (this.python)
+                            return 'python';
+                    })(),
+                    on_message: (buffer, websocket) => {
+                        this.on_message(buffer, websocket);
+                    },
+                    on_error: error => {
+                        this.error ??= new DdbConnectionError(this, error);
+                        this.on_error();
+                    }
+                });
+            }
+            catch (error) {
+                this.error ??= new DdbConnectionError(this, error);
+                reject(this.error);
+                return;
+            }
+            try {
+                assert(this.connected);
+                await this.rpc('connect', { skip_connection_check: true });
+                if (this.autologin)
+                    await this.call('login', [this.username, this.password], { urgent: true, skip_connection_check: true });
+                if (this.streaming)
+                    await this.subscribe();
+                resolve();
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
     }
     get_rpc_options({ urgent = false, secondary = false, async: _async = false, pickle = false, clear = false, api = false, compress = false, cancellable = true, priority = urgent ? 8 : 4, parallelism = 8, root_id = '', limit, } = {}) {
         let flag = 0;
@@ -2367,7 +2353,8 @@ export class DDB {
     }
     disconnect() {
         if (this.connected)
-            this.websocket.close(1000);
+            // 这里不获取 lock，直接关闭连接
+            this.lwebsocket.resource.close(1000);
     }
     /** (内部使用的方法) rpc through websocket (function/script/variable command)
         未连接到 DDB 时调用会自动连接，连接断开时调用会抛出 DdbConnectionError
@@ -2378,165 +2365,138 @@ export class DDB {
             - vars?: type === 'variable' 时必传，variable 指令中待上传的变量名
             - listener?: 处理本次 rpc 期间的消息 (DdbMessage)
             - parse_object?: 在本次 rpc 期间设置 parse_object, 结束后恢复原有
-                为 false 时返回的 DdbObj 仅含有 buffer 和 le，不做解析，以便后续转发、序列化 */
+                为 false 时返回的 DdbObj 仅含有 buffer 和 le，不做解析，以便后续转发、序列化
+            - skip_connection_check?: 在首次 await ddb.connect() 建立连接时不能再次调用 await this.connect() 确保连接状态，会导致循环依赖，
+                将这个 flag 设为 true 跳过连接状态检查 */
     async rpc(type, options) {
         // 保留调用栈信息
         let error = new DdbDatabaseError('', this, type, options);
-        await this.connect();
+        if (!options.skip_connection_check)
+            await this.connect();
         const { script, func, args: _args = [], vars = [], urgent, listener, } = options;
-        if (func === 'pnode_run' && !this.pnode_run_defined) {
-            // 准备进入 pnode_run 临界区，保证并发调用 rpc 时只定义一次 pnode_run
-            const ptail = this.ppnode_run;
-            let ppnode_run = this.ppnode_run = defer();
-            // 避免 ppnode_run.reject 后出现 triggerUncaughtException
-            ppnode_run.catch(() => { });
-            // 如果之前的定义失败了，再定义一次也会失败，直接抛出之前的错误，且安全地退出临界区
-            try {
-                await ptail;
-                // 已进入 pnode_run 临界区
-            }
-            catch (error) {
-                ppnode_run.reject(error);
-                throw error;
-            }
-            if (!this.pnode_run_defined)
-                try {
-                    await this.eval(this.python ?
-                        'def pnode_run (nodes, func_name, args, add_node_alias):\n' +
-                            '    nargs = size(args)\n' +
-                            '    func = funcByName(func_name)\n' +
-                            '    \n' +
-                            '    if not nargs:\n' +
-                            '        return pnodeRun(func, nodes, add_node_alias)\n' +
-                            '    \n' +
-                            '    args_partial = [ ]\n' +
-                            '    args_partial.append(func)\n' +
-                            '    for a in args:\n' +
-                            '        args_partial.append(a)\n' +
-                            '    \n' +
-                            '    return pnodeRun(\n' +
-                            '        unifiedCall(partial, args_partial),\n' +
-                            '        nodes,\n' +
-                            '        add_node_alias\n' +
-                            '    )\n'
-                        :
-                            'def pnode_run (nodes, func_name, args, add_node_alias = true) {\n' +
-                                '    nargs = size(args)\n' +
-                                '    func = funcByName(func_name)\n' +
-                                '    \n' +
-                                '    if (!nargs)\n' +
-                                '        return pnodeRun(func, nodes, add_node_alias)\n' +
-                                '    \n' +
-                                '    args_partial = array(any, 1 + nargs, 1 + nargs)\n' +
-                                '    args_partial[0] = func\n' +
-                                '    args_partial[1:] = args\n' +
-                                '    return pnodeRun(\n' +
-                                '        unifiedCall(partial, args_partial),\n' +
-                                '        nodes,\n' +
-                                '        add_node_alias\n' +
-                                '    )\n' +
-                                '}\n', { urgent: true });
-                    this.pnode_run_defined = true;
-                    ppnode_run.resolve();
-                }
-                catch (error) {
-                    // 这次失败了，之后的执行肯定也会失败
-                    ppnode_run.reject(error);
-                    throw error;
-                }
-        }
+        if (func === 'pnode_run')
+            await (this.ppnode_run ??= this.eval(this.python ?
+                '\n' +
+                    'def pnode_run (nodes, func_name, args, add_node_alias):\n' +
+                    '    nargs = size(args)\n' +
+                    '    func = funcByName(func_name)\n' +
+                    '    \n' +
+                    '    if not nargs:\n' +
+                    '        return pnodeRun(func, nodes, add_node_alias)\n' +
+                    '    \n' +
+                    '    args_partial = [ ]\n' +
+                    '    args_partial.append(func)\n' +
+                    '    for a in args:\n' +
+                    '        args_partial.append(a)\n' +
+                    '    \n' +
+                    '    return pnodeRun(\n' +
+                    '        unifiedCall(partial, args_partial),\n' +
+                    '        nodes,\n' +
+                    '        add_node_alias\n' +
+                    '    )\n'
+                :
+                    // 这个开头的空行很重要，应该可以绕过 webLoginRequired = true 时禁止执行代码
+                    // 搜一下 APISocketConsole::execute
+                    // https://dolphindb1.atlassian.net/browse/D20-4991
+                    '\n' +
+                        'def pnode_run (nodes, func_name, args, add_node_alias = true) {\n' +
+                        '    nargs = size(args)\n' +
+                        '    func = funcByName(func_name)\n' +
+                        '    \n' +
+                        '    if (!nargs)\n' +
+                        '        return pnodeRun(func, nodes, add_node_alias)\n' +
+                        '    \n' +
+                        '    args_partial = array(any, 1 + nargs, 1 + nargs)\n' +
+                        '    args_partial[0] = func\n' +
+                        '    args_partial[1:] = args\n' +
+                        '    return pnodeRun(\n' +
+                        '        unifiedCall(partial, args_partial),\n' +
+                        '        nodes,\n' +
+                        '        add_node_alias\n' +
+                        '    )\n' +
+                        '}\n', { urgent: true }));
         // this 上的当前配置需要在 message 到达后使用，先保存起来
         const _listeners = [...this.listeners].reverse();
-        // websocket 临界区：保证多个 rpc 并发时形成 promise 链
+        // rpc 请求期间需要独占 websocket，所以设计了一个锁，申请之后才能使用
         // ddb 世界观：需要等待上一个 rpc 结果从 server 返回之后才能发起下一个调用  
         // 违反世界观可能造成:  
         // 1. 并发多个请求只返回第一个结果（阻塞，需后续请求疏通）
         // 2. windows 下 ddb server 返回多个相同的结果
-        const ptail = this.pwebsocket;
-        let pwebsocket = this.pwebsocket = defer();
-        // 避免 pwebsocket.reject 后出现 triggerUncaughtException
-        pwebsocket.catch(() => { });
-        // 上一个请求出现了 websocket 错误，那么直接抛出异常，且通过 reject 当前锁 (pwebsocket)，安全退出临界区
-        try {
-            await ptail;
-            // 已进入临界区，只有一个 rpc 函数调用运行到这里，可以独占 this.on_message 然后写 WebSocket
-        }
-        catch (error) {
-            pwebsocket.reject(error);
-            throw error;
-        }
         // 既然上一个请求没有出现 websocket error，且函数开头已经调用了 await this.connect() 检查过，
         // 这里也乐观的认为 this.connected && !this.errored 为 true
-        return new Promise((resolve, reject) => {
-            this.on_error = () => {
-                pwebsocket.reject(this.error /* 这里一定有了 this.error, 不需要再 || new DdbConnectionError(this) */);
-                reject(this.error);
-            };
-            this.on_message = buffer => {
-                try {
-                    const buf = new Uint8Array(buffer);
-                    if (this.print_message_buffer)
-                        console.log(typed_array_to_buffer(buf));
-                    const message = this.parse_message(buf, error);
-                    listener?.(message, this);
-                    for (const listener of _listeners)
-                        listener(message, this);
-                    const { type, data } = message;
+        return this.lwebsocket.request(async (websocket) => {
+            // 独占资源后先检查状态
+            if (this.errored || !this.connected)
+                throw this.error || new DdbConnectionError(this);
+            // 使用资源发送请求并等待请求完成
+            return new Promise((resolve, reject) => {
+                this.on_error = () => {
+                    // 这里一定有了 this.error, 不需要再 || new DdbConnectionError(this)
+                    reject(this.error);
+                };
+                this.on_message = buffer => {
+                    try {
+                        const buf = new Uint8Array(buffer);
+                        if (this.print_message_buffer)
+                            console.log(typed_array_to_buffer(buf));
+                        const message = this.parse_message(buf, error);
+                        listener?.(message, this);
+                        for (const listener of _listeners)
+                            listener(message, this);
+                        const { type, data } = message;
+                        switch (type) {
+                            case 'print':
+                                if (this.print_message)
+                                    console.log(data);
+                                break;
+                            case 'object':
+                                resolve(data);
+                                break;
+                            case 'error':
+                                reject(data);
+                                break;
+                        }
+                    }
+                    catch (error) {
+                        // 这里的错误并非 websocket 错误，而是 rpc 错误
+                        reject(error);
+                    }
+                };
+                const args = DdbObj.to_ddbobjs(_args);
+                const command = this.enc.encode((() => {
                     switch (type) {
-                        case 'print':
-                            if (this.print_message)
-                                console.log(data);
-                            break;
-                        case 'object':
-                            pwebsocket.resolve();
-                            resolve(data);
-                            break;
-                        case 'error':
-                            pwebsocket.resolve();
-                            reject(data);
-                            break;
+                        case 'function':
+                            if (this.verbose)
+                                console.log(func + inspect(args, { colors: false }));
+                            return 'function\n' +
+                                `${func}\n` +
+                                `${args.length}\n` +
+                                `${Number(DDB.le_client)}\n`;
+                        case 'script':
+                            if (this.verbose)
+                                console.log(script);
+                            return 'script\n' +
+                                script;
+                        case 'variable':
+                            if (this.verbose)
+                                for (let i = 0; i < vars.length; i++)
+                                    console.log(`${vars[i]} = ${inspect(args[i], { colors: false })}`);
+                            return 'variable\n' +
+                                `${vars.join(',')}\n` +
+                                `${vars.length}\n` +
+                                `${Number(DDB.le_client)}\n`;
+                        case 'connect':
+                            if (this.verbose)
+                                console.log('connect()');
+                            return 'connect\n';
                     }
-                }
-                catch (error) {
-                    pwebsocket.resolve();
-                    // 这里的错误并非 websocket 错误，而是 rpc 错误
-                    reject(error);
-                }
-            };
-            const args = DdbObj.to_ddbobjs(_args);
-            const command = this.enc.encode((() => {
-                switch (type) {
-                    case 'function':
-                        if (this.verbose)
-                            console.log(func + inspect(args, { colors: false }));
-                        return 'function\n' +
-                            `${func}\n` +
-                            `${args.length}\n` +
-                            `${Number(DDB.le_client)}\n`;
-                    case 'script':
-                        if (this.verbose)
-                            console.log(script);
-                        return 'script\n' +
-                            script;
-                    case 'variable':
-                        if (this.verbose)
-                            for (let i = 0; i < vars.length; i++)
-                                console.log(`${vars[i]} = ${inspect(args[i], { colors: false })}`);
-                        return 'variable\n' +
-                            `${vars.join(',')}\n` +
-                            `${vars.length}\n` +
-                            `${Number(DDB.le_client)}\n`;
-                    case 'connect':
-                        if (this.verbose)
-                            console.log('connect()');
-                        return 'connect\n';
-                }
-            })());
-            this.websocket.send(concat([
-                this.enc.encode(`API2 ${this.sid} ${command.length} / ${this.get_rpc_options({ urgent })}\n`),
-                command,
-                ...args.map(arg => arg.pack())
-            ]));
+                })());
+                websocket.send(concat([
+                    this.enc.encode(`API2 ${this.sid} ${command.length} / ${this.get_rpc_options({ urgent })}\n`),
+                    command,
+                    ...args.map(arg => arg.pack())
+                ]));
+            });
         });
     }
     /** eval script through websocket (script command)
@@ -2576,8 +2536,11 @@ export class DDB {
                 Set parse_object during this rpc, and restore the original after the end.
                 When it is false, the returned DdbObj only contains buffer and le without parsing,
                 so as to facilitate subsequent forwarding and serialization
-    */
-    async call(func, args = [], { urgent, node, nodes, func_type, add_node_alias, listener, parse_object, } = {}) {
+            - skip_connection_check?: (内部使用) 在首次 await ddb.connect() 建立连接时不能再次调用 await this.connect() 确保连接状态，会导致循环依赖，
+                将这个 flag 设为 true 跳过连接状态检查
+                (internal use) When await ddb.connect() establishes a connection for the first time, you cannot call await this.connect() again to ensure the connection status, which will lead to circular dependencies.
+                 Set this flag to true to skip connection status checks */
+    async call(func, args = [], { urgent, node, nodes, func_type, add_node_alias, listener, parse_object, skip_connection_check } = {}) {
         if (node) {
             assert(func_type in DdbFunctionType, t('指定 node 时必须设置 func_type'));
             args = [
@@ -2607,7 +2570,8 @@ export class DDB {
             args,
             urgent,
             listener,
-            parse_object
+            parse_object,
+            skip_connection_check
         });
     }
     /** upload variable through websocket (variable command) */
@@ -2699,7 +2663,7 @@ export class DDB {
             // new DdbInt(-1),  // offset
             // filter
             // allow exists
-        ])).value);
+        ], { skip_connection_check: true })).value);
         this.streaming.window = {
             offset: 0,
             rows: 0,
@@ -2741,6 +2705,7 @@ export class DDB {
         return this.streaming;
     }
 }
+export { DDB };
 // const ddb_gateway = {
 //         server: null as SocketServer,
 //         clients: new Set<Socket>(),