dolphindb 0.0.4 → 0.0.7

package/browser.js

@@ -1,11 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ddb = exports.DDB = exports.DdbFunction = exports.DdbPair = exports.DdbVectorAny = exports.DdbVectorDouble = exports.DdbVectorString = exports.DdbVectorInt = exports.DdbDouble = exports.DdbLong = exports.DdbString = exports.DdbInt = exports.DdbBool = exports.DdbObj = exports.nulls = exports.DdbFunctionType = exports.DdbType = exports.DdbForm = void 0;
+exports.ddb = exports.DDB = exports.timestamp2str = exports.date2str = exports.DdbFunction = exports.DdbPair = exports.DdbVectorAny = exports.DdbVectorDouble = exports.DdbVectorString = exports.DdbVectorInt = exports.DdbDouble = exports.DdbLong = exports.DdbString = exports.DdbInt = exports.DdbBool = exports.DdbVoid = exports.DdbObj = exports.timezone_offset = exports.nulls = exports.DdbFunctionType = exports.DdbType = exports.DdbForm = void 0;
 const tslib_1 = require("tslib");
 const dayjs_1 = (0, tslib_1.__importDefault)(require("dayjs"));
 require("xshell/prototype.browser");
 const utils_browser_1 = require("xshell/utils.browser");
-const chalk_browser_1 = require("xshell/chalk.browser");
 var DdbForm;
 (function (DdbForm) {
     DdbForm[DdbForm["scalar"] = 0] = "scalar";
@@ -75,6 +74,8 @@ exports.nulls = {
     int32: -2147483648,
     int64: -0x8000000000000000n, // -922_3372_0368_5477_5808
 };
+exports.timezone_offset = 1000 * 60 * new Date().getTimezoneOffset();
+/** 可以表示所有 DolphinDB 数据库中的数据类型 */
 class DdbObj {
     constructor(data) {
         /** 是否为小端 (little endian) */
@@ -333,16 +334,86 @@ class DdbObj {
     static parse_vector(buf, le, type) {
         const dv = new DataView(buf.buffer, buf.byteOffset);
         const rows = dv.getUint32(0, le);
-        const i_items_start = 8;
-        const [len_items, value] = this.parse_vector_items(buf.subarray(i_items_start), le, type, rows);
+        let i_items_start = 8;
+        if (type < 64 || type >= 128) {
+            const [len_items, value] = this.parse_vector_items(buf.subarray(i_items_start), le, type, rows);
+            return new this({
+                le,
+                form: DdbForm.vector,
+                type,
+                length: i_items_start + len_items,
+                cols: 1,
+                rows,
+                value,
+            });
+        }
+        // array vector
+        // av = array(INT[], 0, 3)
+        // append!(av, [1..4])
+        // append!(av, [1..70000])
+        // av
+        // <Buffer 44 01  type = array vector, form = vector
+        // 02 00 00 00 74 11 01 00 rows = 2, cols = 70004 (0x011174)
+        // block 0
+        // 01 00 block.rows = 1
+        // 04 block.unit = 4
+        // 00 reserved
+        // 04 00 00 00 block.lengths = [4]
+        // 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 block.data
+        // block 1
+        // 01 00 block.rows = 1
+        // 04 block.unit = 4
+        // 00 reserved
+        // 70 11 01 00 block.lengths = [70000 (0x00011170)]
+        // 01 00 00 00 02 00 00 00 ... 279992 more bytes> block.data
+        const cols = dv.getUint32(4, le);
+        let blocks = [];
+        let i_block_start = i_items_start;
+        // 解析一个 block
+        for (let i_row = 0; i_row < rows;) {
+            /** 对应 array vector 中元素个数 */
+            const rows = dv.getUint16(i_block_start, le);
+            /** 每个 length 占用的字节数 */
+            const unit = dv.getUint8(i_block_start + 2);
+            /** array vector 每个元素的子元素长度 */
+            let lengths;
+            const i_lengths_start = i_block_start + 4;
+            const i_data_start = i_lengths_start + rows * unit;
+            const lengths_buf = buf.slice(i_lengths_start, i_data_start);
+            switch (unit) {
+                case 1:
+                    lengths = lengths_buf;
+                    break;
+                case 2:
+                    lengths = new Uint16Array(lengths_buf.buffer);
+                    break;
+                case 4:
+                    lengths = new Uint32Array(lengths_buf.buffer);
+                    break;
+                default:
+                    throw new Error(`array vector 存在非法 unit = ${unit}`);
+            }
+            let total_length = 0;
+            for (const x of lengths)
+                total_length += x;
+            const [len_items, data] = this.parse_vector_items(buf.subarray(i_data_start), le, type - 64, total_length);
+            blocks.push({
+                unit,
+                rows,
+                lengths,
+                data
+            });
+            i_block_start = i_data_start + len_items;
+            i_row += rows;
+        }
         return new this({
             le,
             form: DdbForm.vector,
             type,
-            length: i_items_start + len_items,
-            cols: 1,
+            length: i_block_start,
+            cols,
             rows,
-            value,
+            value: blocks
         });
     }
     /** 有可能没有字节对齐，不能直接使用原有 message 的 arraybuffer, 统一复制出来，让原有 arraybuffer 被回收掉比较好 */
@@ -423,13 +494,13 @@ class DdbObj {
                 }
                 const value_start = 8 + base_length;
                 const value_end = value_start + length * 4;
-                const value = new Uint32Array(buf.buffer.slice(buf.byteOffset + value_start, buf.byteOffset + value_end));
+                const data = new Uint32Array(buf.buffer.slice(buf.byteOffset + value_start, buf.byteOffset + value_end));
                 return [
                     value_end,
                     {
                         base_id,
                         base,
-                        value
+                        data
                     }
                 ];
             }
@@ -554,11 +625,24 @@ class DdbObj {
                             return [Float64Array.from(this.value)];
                         case DdbType.duration:
                             return [value];
+                        case DdbType.handle:
+                            throw new Error('DolphinDB Server 不支持 handle 的反序列化');
                         default:
                             throw new Error(`${DdbType[type]} 暂时不支持序列化`);
                     }
                 case DdbForm.vector:
                 case DdbForm.pair:
+                    // pack array vector
+                    if (form === DdbForm.vector && 64 <= type && type < 128)
+                        return [
+                            Uint32Array.of(this.rows, this.cols),
+                            ...this.value.map(block => ([
+                                Uint16Array.of(block.rows),
+                                Uint8Array.of(block.unit, 0),
+                                block.lengths,
+                                block.data
+                            ])).flat()
+                        ];
                     return [
                         Uint32Array.of(this.rows, 1),
                         ...DdbObj.pack_vector_body(value, type, this.rows)
@@ -674,6 +758,14 @@ class DdbObj {
                     bufs[i] = value[i].pack();
                 return bufs;
             }
+            case DdbType.symbol_extended: {
+                const { base_id, base, data } = value;
+                return [
+                    Uint32Array.of(base_id, base.length),
+                    ...this.pack_vector_body(base, DdbType.string, base.length),
+                    data
+                ];
+            }
             default:
                 throw new Error(`vector ${DdbType[type]} 暂不支持序列化`);
         }
@@ -687,6 +779,8 @@ class DdbObj {
                         return `functiondef<${DdbFunctionType[this.value.type]}>`;
                     return tname;
                 case DdbForm.vector:
+                    if (64 <= this.type && this.type < 128)
+                        return `${DdbType[this.type - 64]}[][${this.rows}]`;
                     return `${tname}[${this.rows}]`;
                 case DdbForm.pair:
                     return `pair<${tname}>`;
@@ -709,7 +803,7 @@ class DdbObj {
                 return `'${this.value.name}'`;
             return this.value;
         })();
-        return `${(0, chalk_browser_1.blue)(type)}(${this.name ? `'${(0, chalk_browser_1.yellow)(this.name)}', ` : ''}${data})\n`;
+        return `${type}(${this.name ? `'${this.name}', ` : ''}${data})\n`;
     }
     to_cols() {
         return this.value.map(col => {
@@ -719,18 +813,10 @@ class DdbObj {
             };
             switch (col.type) {
                 case DdbType.timestamp:
-                    col_.render = (value) => {
-                        if (value === exports.nulls.int64)
-                            return '';
-                        return (0, dayjs_1.default)(Number(value)).format('YYYY.MM.DD HH:mm:ss.SSS');
-                    };
+                    col_.render = timestamp2str;
                     break;
                 case DdbType.date:
-                    col_.render = (value) => {
-                        if (value === exports.nulls.int32)
-                            return '';
-                        return (0, dayjs_1.default)(Number(1000 * 3600 * 24 * value)).format('YYYY.MM.DD');
-                    };
+                    col_.render = date2str;
                     break;
                 case DdbType.ipaddr:
                     col_.render = value => value.slice().reverse().join('.').replace(/^(0\.)+/, '');
@@ -755,6 +841,15 @@ class DdbObj {
         }
         return rows;
     }
+    to_dict() {
+        if (this.form !== DdbForm.dict)
+            throw new Error('this.form 不是 dict, 不能转换为 Object');
+        const [{ value: keys }, { value: values }] = this.value;
+        let obj = {};
+        for (let i = 0; i < this.rows; i++)
+            obj[keys[i]] = values[i].value;
+        return obj;
+    }
 }
 exports.DdbObj = DdbObj;
 DdbObj.dec = new TextDecoder('utf-8');
@@ -763,6 +858,16 @@ DdbObj.enc = new TextEncoder();
 DdbObj.symbol_bases = {};
 /** little endian (client) */
 DdbObj.le_client = Boolean(new Uint8Array(Uint32Array.of(1).buffer)[0]);
+class DdbVoid extends DdbObj {
+    constructor() {
+        super({
+            form: DdbForm.scalar,
+            type: DdbType.void,
+            length: 0,
+        });
+    }
+}
+exports.DdbVoid = DdbVoid;
 class DdbBool extends DdbObj {
     constructor(value) {
         super({
@@ -894,95 +999,162 @@ class DdbFunction extends DdbObj {
     }
 }
 exports.DdbFunction = DdbFunction;
+function date2str(date) {
+    return date === exports.nulls.int32 ?
+        ''
+        :
+            (0, dayjs_1.default)(exports.timezone_offset + 1000 * 3600 * 24 * date).format('YYYY.MM.DD');
+}
+exports.date2str = date2str;
+function timestamp2str(timestamp) {
+    return timestamp === exports.nulls.int64 ?
+        ''
+        :
+            (0, dayjs_1.default)(exports.timezone_offset + Number(timestamp)).format('YYYY.MM.DD HH:mm:ss.SSS');
+}
+exports.timestamp2str = timestamp2str;
 class DDB {
-    constructor(ws_url) {
+    /**
+        Initialize an instance of DolphinDB Client using the WebSocket URL
+        (without establishing an actual network connection)
+        @example
+        let ddb = new DDB('ws://127.0.0.1:8848')
+        let ddb_secure = new DDB('wss://dolphindb.com')
+    */
+    constructor(url) {
         /** 当前的 session id (http 或 tcp) */
         this.sid = '0';
         /** utf-8 text decoder */
         this.dec = new TextDecoder('utf-8');
         this.enc = new TextEncoder();
-        this.ws = null;
+        this.websocket = null;
         /** little endian (server) */
         this.le = true;
+        /** python session flag (2048) */
+        this.python = false;
+        this.message_hook = null;
         this.presult = Promise.resolve(null);
-        if (!ws_url) {
-            const url = new URL(location.href);
-            const hostname = url.searchParams.get('hostname') || location.hostname;
-            const port = url.searchParams.get('port') || location.port;
-            ws_url = `${location.protocol === 'https:' ? 'wss' : 'ws'}://${hostname}${port ? `:${port}` : ''}/`;
+        if (!url) {
+            const _url = new URL(location.href);
+            const hostname = _url.searchParams.get('hostname') || location.hostname;
+            const port = _url.searchParams.get('port') || location.port;
+            url = `${location.protocol === 'https:' ? 'wss' : 'ws'}://${hostname}${port ? `:${port}` : ''}/`;
         }
-        this.ws_url = ws_url;
+        this.url = url;
+    }
+    /** print message handler */
+    printer(message) {
+        console.log(message);
     }
     /** resolver, rejector, promise of last rpc */
     presolver(buf) { }
     prejector(error) { }
-    /** 连接到 DolphinDB Server */
-    async connect({ ws_url = this.ws_url, login = true, username = 'admin', password = '123456', } = {}) {
-        this.ws_url = ws_url;
-        if (this.ws?.readyState === WebSocket.OPEN)
+    /** Establish the actual WebSocket connection to the DolphinDB corresponding to the URL */
+    async connect({ url = this.url, login = true, username = 'admin', password = '123456', python = false, } = {}) {
+        this.url = url;
+        this.python = python;
+        if (this.websocket?.readyState === WebSocket.OPEN)
             this.disconnect();
-        let ws = new WebSocket(ws_url);
+        let websocket = new WebSocket(url, python ? ['python'] : []);
         // https://stackoverflow.com/questions/11821096/what-is-the-difference-between-an-arraybuffer-and-a-blob/39951543
-        ws.binaryType = 'arraybuffer';
-        this.ws = ws;
-        ws.addEventListener('open', ev => {
-            console.log(`${this.ws_url} opened`);
-            ws.send(`API ${this.sid} 8\n` +
-                'connect\n');
-        });
-        ws.addEventListener('close', ev => {
-            console.log(`${this.ws_url} closed with code = ${ev.code}, reason = '${ev.reason}'`);
-        });
-        ws.addEventListener('error', ev => {
-            console.log(`${this.ws_url} errored`, ev);
-        });
-        // 为首个 connect 响应报文初始化 presult 链表头结点
-        this.presult = new Promise((resolve, reject) => {
-            this.presolver = resolve;
-            this.prejector = reject;
-        });
-        ws.addEventListener('message', ({ data: buf }) => {
-            try {
-                this.presolver(this.parse_message(new Uint8Array(buf)));
-            }
-            catch (error) {
-                this.prejector(error);
-            }
+        websocket.binaryType = 'arraybuffer';
+        this.websocket = websocket;
+        await new Promise((resolve, reject) => {
+            websocket.addEventListener('open', async (event) => {
+                console.log(`${websocket.url} opened`);
+                await this.rpc('connect', {});
+                resolve();
+            });
+            websocket.addEventListener('close', event => {
+                console.log(`${websocket.url} closed with code = ${event.code}, reason = '${event.reason}'`);
+            });
+            websocket.addEventListener('error', event => {
+                const message = `${websocket.url} errored`;
+                console.error(message, event);
+                reject(Object.assign(new Error(message), { event }));
+            });
+            websocket.addEventListener('message', event => {
+                const buf = new Uint8Array(event.data);
+                if (this.message_hook)
+                    this.message_hook(buf);
+                try {
+                    const { type, data } = this.parse_message(buf);
+                    if (type === 'message') {
+                        this.printer(data);
+                        return;
+                    }
+                    this.presolver(data);
+                }
+                catch (error) {
+                    this.prejector(error);
+                }
+            });
         });
-        await this.presult;
-        await this.eval('def pnode_run (nodes, func_name, args, add_node_alias = true) {\n' +
-            '    nargs = size(args)\n' +
-            '    func = funcByName(func_name)\n' +
-            '    \n' +
-            '    if (!nargs)\n' +
-            '        return pnodeRun(func, nodes, add_node_alias)\n' +
-            '    \n' +
-            '    args_partial = array(any, 1 + nargs, 1 + nargs)\n' +
-            '    args_partial[0] = func\n' +
-            '    args_partial[1:] = args\n' +
-            '    return pnodeRun(\n' +
-            '        unifiedCall(partial, args_partial),\n' +
-            '        nodes,\n' +
-            '        add_node_alias\n' +
-            '    )\n' +
-            '}\n', { urgent: true });
+        if (!this.python)
+            await this.eval('def pnode_run (nodes, func_name, args, add_node_alias = true) {\n' +
+                '    nargs = size(args)\n' +
+                '    func = funcByName(func_name)\n' +
+                '    \n' +
+                '    if (!nargs)\n' +
+                '        return pnodeRun(func, nodes, add_node_alias)\n' +
+                '    \n' +
+                '    args_partial = array(any, 1 + nargs, 1 + nargs)\n' +
+                '    args_partial[0] = func\n' +
+                '    args_partial[1:] = args\n' +
+                '    return pnodeRun(\n' +
+                '        unifiedCall(partial, args_partial),\n' +
+                '        nodes,\n' +
+                '        add_node_alias\n' +
+                '    )\n' +
+                '}\n', { urgent: true });
         if (login)
             await this.call('login', [username, password], { urgent: true });
     }
+    get_rpc_options({ urgent = false, secondary = false, async: _async = false, pickle = false, clear = false, api = false, compress = false, cancellable = true, priority = urgent ? 8 : 4, parallelism = 8, root_id = '', limit, } = {}) {
+        let flag = 0;
+        if (urgent)
+            flag += 1;
+        if (secondary)
+            flag += 2;
+        if (_async)
+            flag += 4;
+        if (pickle)
+            flag += 8;
+        if (clear)
+            flag += 16;
+        if (api)
+            flag += 32;
+        if (compress)
+            flag += 64;
+        // python session
+        if (this.python)
+            flag += 2048;
+        const options = [
+            flag,
+            cancellable ? 1 : 0,
+            priority,
+            parallelism,
+            ...limit ? [
+                root_id,
+                limit,
+            ] : [],
+        ];
+        return `/ ${options.join('_')}`;
+    }
     disconnect() {
-        this.ws?.close(1000);
+        this.websocket?.close(1000);
     }
-    /** rpc through websocket (function command)
+    /** rpc through websocket (function/script/variable command)
         - type: API 类型: 'script' | 'function' | 'variable'
         - options:
             - urgent?: 决定 `行为标识` 那一行字符串的取值（只适用于 script 和 function）
             - vars?: type === 'variable' 时必传，variable 指令中待上传的变量名
     */
     async rpc(type, { script, func, args = [], vars = [], urgent, }) {
-        if (!this.ws)
+        if (!this.websocket)
             await this.connect();
-        if (this.ws.readyState !== WebSocket.OPEN)
-            throw new Error('ws 连接已断开');
+        if (this.websocket.readyState !== WebSocket.OPEN)
+            throw new Error(`${this.url} is already disconnected`);
         // 临界区：保证多个 rpc 并发时形成 promise 链
         // ddb 世界观：需要等待上一个 rpc 结果从 server 返回之后才能发起下一个调用  
         // 违反世界观可能造成:  
@@ -1020,14 +1192,16 @@ class DDB {
                         `${vars.join(',')}\n` +
                         `${vars.length}\n` +
                         `${Number(DDB.le_client)}\n`;
+                case 'connect':
+                    return 'connect\n';
             }
         })());
         const message = (0, utils_browser_1.concat)([
-            this.enc.encode(`API ${this.sid} ${command.length}${urgent ? ` / 1_1_8_8` : ''}\n`),
+            this.enc.encode(`API2 ${this.sid} ${command.length}${this.get_rpc_options({ urgent })}\n`),
             command,
             ...args.map((arg) => arg.pack())
         ]);
-        this.ws.send(message);
+        this.websocket.send(message);
         return DdbObj.parse(await presult, // data_buf
         this.le);
     }
@@ -1043,13 +1217,15 @@ class DDB {
         - func: 函数名
         - args?: `[ ]` 调用参数 (传入的原生 string 和 boolean 会被自动转换为 DdbObj<string> 和 DdbObj<boolean>)
         - options?: 调用选项
-            - urgent?: 紧急 flag，使用 urgent worker 处理，防止被其它作业阻塞
+            - urgent?: 紧急 flag。使用 urgent worker 执行，防止被其它作业阻塞
             - node?: 设置结点 alias 时发送到集群中对应的结点执行 (使用 DolphinDB 中的 rpc 方法)
             - nodes?: 设置多个结点 alias 时发送到集群中对应的多个结点执行 (使用 DolphinDB 中的 pnodeRun 方法)
             - func_type?: 设置 node 参数时必传，需指定函数类型，其它情况下不传
             - add_node_alias?: 设置 nodes 参数时选传，其它情况不传
     */
     async call(func, args = [], { urgent, node, nodes, func_type, add_node_alias } = {}) {
+        if (this.python)
+            throw new Error('call doesn\'t support python session yet');
         if (node) {
             if (typeof func_type === 'undefined')
                 throw new Error('指定 node 时必须设置 func_type');
@@ -1065,7 +1241,13 @@ class DDB {
                 new DdbVectorString(nodes),
                 func,
                 new DdbVectorAny(this.to_ddbobjs(args)),
-                ...typeof add_node_alias === 'undefined' ? [] : [add_node_alias]
+                ...(() => {
+                    if (typeof add_node_alias !== 'undefined')
+                        return [add_node_alias];
+                    if (this.python)
+                        return [true];
+                    return [];
+                })()
             ];
             func = 'pnode_run';
         }
@@ -1077,9 +1259,9 @@ class DDB {
     }
     /** upload variable through websocket (variable command) */
     async upload(
-    /** 上传的变量名 */
+    /** Uploaded variables' name */
     vars, 
-    /** 上传的变量值 */
+    /** Uploaded variables' value */
     args) {
         if (!args.length || args.length !== vars.length)
             throw new Error('variable 指令参数为空或参数名为空，或数量不匹配');
@@ -1087,6 +1269,14 @@ class DDB {
     }
     /** 解析服务端响应报文，返回去掉 header 的 data buf */
     parse_message(buf) {
+        // MSG\n
+        // <message>\0
+        // 'M'.codePointAt(0).to_hex_str()
+        if (buf[0] === 0x4d && buf[1] === 0x53 && buf[2] === 0x47 && buf[3] === 0x0a)
+            return {
+                type: 'message',
+                data: this.dec.decode(buf.subarray(4))
+            };
         // '1166953221 1 1\n'
         // 'OK\n'
         // '\x04\x00\x02\x00\x00\x00'
@@ -1109,7 +1299,10 @@ class DDB {
         const message = this.dec.decode(buf.subarray(i_ls_1, i_lf_1));
         if (message !== 'OK')
             throw new Error(message);
-        return buf.subarray(i_lf_1 + 1);
+        return {
+            type: 'object',
+            data: buf.subarray(i_lf_1 + 1)
+        };
     }
     /** 自动转换 js string, boolean 为 DdbObj */
     to_ddbobj(value) {