dolphindb 2.0.801 → 2.0.901

package/browser.js

@@ -118,15 +118,6 @@ export const nulls = {
     double: -Number.MAX_VALUE,
     bytes16: Uint8Array.from(new Array(16).fill(0))
 };
-export const timezone_offset = 1000 * 60 * new Date().getTimezoneOffset();
-export var StreamingStatusCode;
-(function (StreamingStatusCode) {
-    StreamingStatusCode[StreamingStatusCode["ok"] = 0] = "ok";
-    StreamingStatusCode[StreamingStatusCode["connection_existed"] = 1] = "connection_existed";
-    StreamingStatusCode[StreamingStatusCode["not_leader"] = 2] = "not_leader";
-    StreamingStatusCode[StreamingStatusCode["other"] = 3] = "other";
-    StreamingStatusCode[StreamingStatusCode["error"] = 4] = "error";
-})(StreamingStatusCode = StreamingStatusCode || (StreamingStatusCode = {}));
 /** 可以表示所有 DolphinDB 数据库中的数据类型 */
 export class DdbObj {
     static dec = new TextDecoder('utf-8');
@@ -522,8 +513,37 @@ export class DdbObj {
         // 00 reserved
         // 70 11 01 00 block.lengths = [70000 (0x00011170)]
         // 01 00 00 00 02 00 00 00 ... 279992 more bytes> block.data
+        // decimal array vector
+        // a = array(DECIMAL32(2)[], 0, 10)
+        // append!(a, [
+        //     [1, 2, 3], 
+        //     [4, 5], 
+        //     [6, 7, 8],
+        //     [9, 10]
+        // ])
+        // print(a)
+        // a
+        // [[1.00,2.00,3.00],[4.00,5.00],[6.00,7.00,8.00],[9.00,10.00]]
+        // <Buffer 65 01 type = 101 = 64 + 37 (decimal32) , form = vector
+        // 04 00 00 00 0a 00 00 00 rows = 4, cols = 10
+        // scale (只有 decimal32 才有)
+        // 02 00 00 00 scale = 2
+        // block 0
+        // 04 00 block.rows = 4
+        // 01 block.unit = 1
+        // 00 reserved
+        // 03 02 03 02 block.lengths = [3, 2, 3, 2]
+        // 64 00 00 00 c8 00 00 00 2c 01 00 00 90 01 00 00 f4 01 00 00 58 02 00 00 bc 02 00 00 ... 12 more bytes> [100, 200, 300, ...]
+        // block 1
+        // ...
+        const type_ = type - 64;
         const cols = dv.getUint32(4, le);
         let blocks = [];
+        // decimal32/64 会在所有 blocks 之前多一个 scale
+        if (type_ === DdbType.decimal32 || type_ === DdbType.decimal64) {
+            blocks.scale = dv.getInt32(i_items_start, le);
+            i_items_start += 4;
+        }
         let i_block_start = i_items_start;
         // 解析一个 block
         for (let i_row = 0; i_row < rows;) {
@@ -552,7 +572,20 @@ export class DdbObj {
             let total_length = 0;
             for (const x of lengths)
                 total_length += x;
-            const [len_items, data] = this.parse_vector_items(buf.subarray(i_data_start), le, type - 64, total_length);
+            let len_items;
+            let data;
+            switch (type_) {
+                case DdbType.decimal32:
+                    len_items = total_length * 4;
+                    data = new Int32Array(buf.buffer.slice(buf.byteOffset + i_data_start, buf.byteOffset + i_data_start + len_items));
+                    break;
+                case DdbType.decimal64:
+                    len_items = total_length * 8;
+                    data = new BigInt64Array(buf.buffer.slice(buf.byteOffset + i_data_start, buf.byteOffset + i_data_start + len_items));
+                    break;
+                default:
+                    [len_items, data] = this.parse_vector_items(buf.subarray(i_data_start), le, type - 64, total_length);
+            }
             blocks.push({
                 unit,
                 rows,
@@ -867,6 +900,10 @@ export class DdbObj {
                     if (form === DdbForm.vector && 64 <= type && type < 128)
                         return [
                             Uint32Array.of(this.rows, this.cols),
+                            ...(type - 64 === DdbType.decimal32 || type - 64 === DdbType.decimal64) ?
+                                [Int32Array.of(this.value.scale)]
+                                :
+                                    [],
                             ...this.value.map(block => ([
                                 Uint16Array.of(block.rows),
                                 Uint8Array.of(block.unit, 0),
@@ -925,10 +962,7 @@ export class DdbObj {
         })();
         if (!body)
             return new Uint8Array(0);
-        return concat([
-            header,
-            ...body
-        ]);
+        return concat([header, ...body]);
     }
     static pack_vector_body(value, type, length) {
         switch (type) {
@@ -1054,7 +1088,18 @@ export class DdbObj {
                             for (const length of lengths) {
                                 let _items = new Array(Math.min(limit, length));
                                 for (let i = 0; i < _items.length; i++)
-                                    _items[i] = format(_type, data[acc_len + i], this.le, options);
+                                    if (_type === DdbType.decimal32 || _type === DdbType.decimal64) {
+                                        const x = data[acc_len + i];
+                                        if (x === nulls.int64 && _type === DdbType.decimal64 ||
+                                            x === nulls.int32 && _type === DdbType.decimal32)
+                                            return '';
+                                        const { scale } = this.value;
+                                        const s = String(x < 0 ? -x : x).padStart(scale, '0');
+                                        const str = (x < 0 ? '-' : '') + (scale ? `${s.slice(0, -scale) || '0'}.${s.slice(-scale)}` : s);
+                                        _items[i] = options.colors ? green(str) : str;
+                                    }
+                                    else
+                                        _items[i] = format(_type, data[acc_len + i], this.le, options);
                                 items[i_items++] = format_array(_items, length > limit);
                                 acc_len += length;
                             }
@@ -1436,6 +1481,7 @@ export function format(type, value, le, options = {}) {
 }
 /** formatted vector, the index-th item in the collection is a string */
 export function formati(obj, index, options = {}) {
+    assert(index < obj.rows, 'index < obj.rows');
     if (64 <= obj.type && obj.type < 128) { // array vector
         // 因为 array vector 目前只支持：Logical, Integral（不包括 INT128, COMPRESS 类型）, Floating, Temporal
         // 都对应 TypedArray 中的一格，所以 lengths.length 等于 block 中的 row 的个数
@@ -1463,7 +1509,18 @@ export function formati(obj, index, options = {}) {
                 const limit = 10;
                 let items = new Array(Math.min(limit, length));
                 for (let i = 0; i < items.length; i++)
-                    items[i] = format(_type, data[acc_len + i], obj.le, options);
+                    if (_type === DdbType.decimal32 || _type === DdbType.decimal64) {
+                        const x = data[acc_len + i];
+                        if (x === nulls.int64 && _type === DdbType.decimal64 ||
+                            x === nulls.int32 && _type === DdbType.decimal32)
+                            return '';
+                        const { scale } = obj.value;
+                        const s = String(x < 0 ? -x : x).padStart(scale, '0');
+                        const str = (x < 0 ? '-' : '') + (scale ? `${s.slice(0, -scale) || '0'}.${s.slice(-scale)}` : s);
+                        items[i] = options.colors ? green(str) : str;
+                    }
+                    else
+                        items[i] = format(_type, data[acc_len + i], obj.le, options);
                 return (items.join(', ') + (length > limit ? ', ...' : '')).bracket('square');
             }
         }
@@ -1799,10 +1856,14 @@ export class DdbTable extends DdbObj {
     }
 }
 export function date2ms(date) {
+    // 将 server 的本地时间 (以 ms 为单位，1970.01.01 00:00:00 作为零点) 当成是客户端这里的本地的时间，然后根据本地的时区信息转换为 utc 时间
+    // 本地的时区与实际的时间值相关，timezone offset 可能会受到夏令时 (DST) 的影响
+    // 得到的 utc 毫秒数交给 js date 或者 dayjs 去格式化
+    const ms = 1000 * 3600 * 24 * date;
     return (date === null || date === nulls.int32) ?
         null
         :
-            timezone_offset + 1000 * 3600 * 24 * date;
+            1000 * 60 * new Date(ms).getTimezoneOffset() + ms;
 }
 export function date2str(date, format = 'YYYY.MM.DD') {
     return (date === null || date === nulls.int32) ?
@@ -1829,7 +1890,7 @@ export function time2ms(time) {
     return (time === null || time === nulls.int32) ?
         null
         :
-            timezone_offset + time;
+            1000 * 60 * new Date(time).getTimezoneOffset() + time;
 }
 export function time2str(time, format = 'HH:mm:ss.SSS') {
     return (time === null || time === nulls.int32) ?
@@ -1838,10 +1899,11 @@ export function time2str(time, format = 'HH:mm:ss.SSS') {
             dayjs(time2ms(time)).format(format);
 }
 export function minute2ms(minute) {
+    const ms = 60 * 1000 * minute;
     return (minute === null || minute === nulls.int32) ?
         null
         :
-            timezone_offset + 60 * 1000 * minute;
+            1000 * 60 * new Date(ms).getTimezoneOffset() + ms;
 }
 export function minute2str(minute, format = 'HH:mm[m]') {
     return (minute === null || minute === nulls.int32) ?
@@ -1850,10 +1912,11 @@ export function minute2str(minute, format = 'HH:mm[m]') {
             dayjs(minute2ms(minute)).format(format);
 }
 export function second2ms(second) {
+    const ms = 1000 * second;
     return (second === null || second === nulls.int32) ?
         null
         :
-            timezone_offset + 1000 * second;
+            1000 * 60 * new Date(ms).getTimezoneOffset() + ms;
 }
 export function second2str(second, format = 'HH:mm:ss') {
     return (second === null || second === nulls.int32) ?
@@ -1862,10 +1925,11 @@ export function second2str(second, format = 'HH:mm:ss') {
             dayjs(second2ms(second)).format(format);
 }
 export function datetime2ms(datetime) {
+    const ms = 1000 * datetime;
     return (datetime === null || datetime === nulls.int32) ?
         null
         :
-            timezone_offset + 1000 * datetime;
+            1000 * 60 * new Date(ms).getTimezoneOffset() + ms;
 }
 export function datetime2str(datetime, format = 'YYYY.MM.DD HH:mm:ss') {
     return (datetime === null || datetime === nulls.int32) ?
@@ -1874,10 +1938,11 @@ export function datetime2str(datetime, format = 'YYYY.MM.DD HH:mm:ss') {
             dayjs(datetime2ms(datetime)).format(format);
 }
 export function timestamp2ms(timestamp) {
+    const ms = Number(timestamp);
     return (timestamp === null || timestamp === nulls.int64) ?
         null
         :
-            timezone_offset + Number(timestamp);
+            1000 * 60 * new Date(ms).getTimezoneOffset() + ms;
 }
 /** format timestamp (bigint) to string
     - timestamp: bigint value
@@ -1892,16 +1957,18 @@ export function timestamp2str(timestamp, format = 'YYYY.MM.DD HH:mm:ss.SSS') {
             dayjs(timestamp2ms(timestamp)).format(format);
 }
 export function datehour2ms(datehour) {
+    const ms = 1000 * 3600 * datehour;
     return (datehour === null || datehour === nulls.int32) ?
         null
         :
-            timezone_offset + 1000 * 3600 * datehour;
+            1000 * 60 * new Date(ms).getTimezoneOffset() + ms;
 }
 export function datehour2str(datehour, format = 'YYYY.MM.DDTHH') {
+    const ms = 1000 * 3600 * datehour;
     return (datehour === null || datehour === nulls.int32) ?
         'null'
         :
-            dayjs(timezone_offset + 1000 * 3600 * datehour).format(format);
+            dayjs(1000 * 60 * new Date(ms).getTimezoneOffset() + ms).format(format);
 }
 /** parse timestamp string to bigint value
     - str: timestamp string, If it is an empty string or 'null', it will return the corresponding empty value (nulls.int64)
@@ -1912,9 +1979,10 @@ export function datehour2str(datehour, format = 'YYYY.MM.DDTHH') {
 export function str2timestamp(str, format = 'YYYY.MM.DD HH:mm:ss.SSS') {
     if (!str || str === 'null')
         return nulls.int64;
-    assert('The length of the timestamp string must be equal to the length of the format string');
-    return BigInt(-timezone_offset +
-        dayjs(str, format).valueOf());
+    assert(str.length === format.length, 'The length of the timestamp string must be equal to the length of the format string');
+    const ms = dayjs(str, format).valueOf();
+    return BigInt(-(1000 * 60 * new Date(ms).getTimezoneOffset()) +
+        ms);
 }
 export function nanotime2ns(nanotime) {
     return nanotimestamp2ns(nanotime);
@@ -1929,15 +1997,17 @@ export function nanotime2str(nanotime, format = 'HH:mm:ss.SSSSSSSSS') {
     const i_second_end = i_second_start + 2;
     const i_nanosecond_start = format.indexOf('SSSSSSSSS', i_second_end);
     assert(i_nanosecond_start !== -1, 'Format string must contain nanosecond format (SSSSSSSSS)');
-    return (dayjs(timezone_offset + (Number(nanotime) / 1000000)).format(format.slice(0, i_second_end)) +
+    const ms = Number(nanotime) / 1000000;
+    return (dayjs(1000 * 60 * new Date(ms).getTimezoneOffset() + ms).format(format.slice(0, i_second_end)) +
         format.slice(i_second_end, i_nanosecond_start) +
         String(nanotime % 1000000000n).padStart(9, '0'));
 }
 export function nanotimestamp2ns(nanotimestamp) {
+    const ms = Number(nanotimestamp / 1000000n);
     return (nanotimestamp === null || nanotimestamp === nulls.int64) ?
         null
         :
-            BigInt(timezone_offset) * 1000000n + nanotimestamp;
+            BigInt(1000 * 60 * new Date(ms).getTimezoneOffset()) * 1000000n + nanotimestamp;
 }
 /** format nanotimestamp value (bigint) to string
     - nanotimestamp: bigint value
@@ -1962,7 +2032,8 @@ export function nanotimestamp2str(nanotimestamp, format = 'YYYY.MM.DD HH:mm:ss.S
     assert(i_nanosecond_start !== -1, 'Format string must contain nanosecond format (SSSSSSSSS)');
     const remainder = nanotimestamp % 1000000000n;
     const borrow = remainder < 0n;
-    return (dayjs(timezone_offset +
+    const ms = Number(nanotimestamp / 1000000n);
+    return (dayjs(1000 * 60 * new Date(ms).getTimezoneOffset() +
         // 去掉 9 位的纳秒部分，转化为毫秒
         Number((nanotimestamp - remainder + (borrow ? -1000000000n : 0n)) / 1000000n)).format(format.slice(0, i_second_end)) +
         format.slice(i_second_end, i_nanosecond_start) +
@@ -1987,8 +2058,9 @@ export function str2nanotimestamp(str, format = 'YYYY.MM.DD HH:mm:ss.SSSSSSSSS')
     const i_second_end = i_second_start + 2;
     const i_nanosecond_start = format.indexOf('SSSSSSSSS', i_second_end);
     assert(i_nanosecond_start !== -1, 'Format string must contain nanosecond format (SSSSSSSSS)');
-    return (BigInt(-timezone_offset +
-        dayjs(str.slice(0, i_second_end), format.slice(0, i_second_end)).valueOf()) * 1000000n
+    const ms = dayjs(str.slice(0, i_second_end), format.slice(0, i_second_end)).valueOf();
+    return (BigInt(-(1000 * 60 * new Date(ms).getTimezoneOffset()) +
+        ms) * 1000000n
         +
             BigInt(str.slice(i_nanosecond_start, i_nanosecond_start + 9)));
 }
@@ -2014,7 +2086,7 @@ export function int1282str(buffer, le = true) {
     return [...buf].map(x => x.toString(16)
         .padStart(2, '0')).join('');
 }
-export const winsize = 40;
+export const winsize = 10_0000;
 export class ConnectionError extends Error {
     ddb;
     constructor(ddb) {
@@ -2086,9 +2158,10 @@ export class DDB {
     constructor(url, options = {}) {
         if (!url) {
             const _url = new URL(location.href);
-            const hostname = _url.searchParams.get('hostname') || location.hostname;
-            const port = _url.searchParams.get('port') || location.port;
-            url = `${location.protocol === 'https:' ? 'wss' : 'ws'}://${hostname}:${port}/`;
+            const dev = _url.pathname.endsWith('/console/');
+            const hostname = _url.searchParams.get('hostname') || (dev ? '127.0.0.1' : location.hostname);
+            const port = _url.searchParams.get('port') || (dev ? '8848' : location.port);
+            url = `${dev ? (_url.searchParams.get('tls') === '1' ? 'wss' : 'ws') : (location.protocol === 'https:' ? 'wss' : 'ws')}://${hostname}${port ? `:${port}` : ''}/`;
         }
         this.url = url;
         if (options.autologin !== undefined)
@@ -2393,8 +2466,9 @@ export class DDB {
     async cancel() {
         let ddb = new DDB(this.url, this);
         try {
+            // 因为是新建的连接，而且执行完脚本之后马上就关闭了，所以不用考虑变量泄漏的问题
             await ddb.eval(`jobs = exec rootJobId from getConsoleJobs() where sessionId = ${this.sid}\n` +
-                'if (size(jobs))\n' +
+                (this.python ? 'if size(jobs):\n' : 'if (size(jobs))\n') +
                 '    cancelConsoleJob(jobs)\n', { urgent: true });
         }
         finally {
@@ -2455,110 +2529,58 @@ export class DDB {
     }
     /** Internal stream subscription method */
     async subscribe() {
-        const args = DdbObj.to_ddbobjs([
-            new DdbVectorString(this.autologin ?
-                [this.username, this.password]
-                :
-                    // 不自动登录也要传空的 string vector, 否则 server 无法解析
-                    []),
-            // todo: 还有用吗？需要获取客户端 IP 吗
+        await this.rpc('connect', {});
+        if (this.autologin)
+            await this.call('login', [this.username, this.password]);
+        console.log('Subscribed to streaming table, colnames:', 
+        // string[3](['time', 'stock', 'price'])
+        this.streaming.colnames = (await this.call('publishTable', [
             'localhost',
-            // todo: 端口还有用吗？
             new DdbInt(0),
             this.streaming.table,
             (this.streaming.action ||= `api_js_${new Date().getTime()}`),
             // new DdbInt(-1),  // offset
             // filter
             // allow exists
-        ]);
-        return new Promise((resolve, reject) => {
-            let first = true;
-            let second = true;
-            // 先准备好收到 websocket message 的 callback
-            this.on_message = ({ data: buffer }) => {
-                if (first) {
-                    first = false;
-                    try {
-                        const dv = new DataView(buffer);
-                        const status = dv.getUint8(1);
-                        this.le = Boolean(dv.getUint8(2));
-                        const nobjs = dv.getUint32(3, this.le);
-                        // 解析返回数据，从 server 实现看其实只有一个 DdbObj
-                        let offset = 7;
-                        let objs = new Array(nobjs);
-                        const buf = new Uint8Array(buffer);
-                        for (let i = 0; i < nobjs; i++) {
-                            const obj = DdbObj.parse(buf.subarray(offset), this.le);
-                            offset += obj.length;
-                            objs[i] = obj;
-                            console.log(`订阅流表的响应数据: objs[${i}]: ${obj.toString()}`);
-                        }
-                        this.streaming.colnames = objs[0].value;
-                        if (status === StreamingStatusCode.ok)
-                            console.log('Subscribed to streaming table:', { status: StreamingStatusCode[status], result: objs[0] });
-                        else
-                            reject(new Error(`Failed to subscribe to streaming table: { status: ${StreamingStatusCode[status]}, error: '${objs[0]?.value}' }`));
-                    }
-                    catch (error) {
-                        reject(error);
-                    }
-                }
-                else {
-                    try {
-                        const dv = new DataView(buffer);
-                        const buf = new Uint8Array(buffer);
-                        let { streaming } = this;
-                        streaming.time = dv.getBigInt64(1, this.le);
-                        streaming.id = dv.getBigInt64(9, this.le);
-                        const i_topic_end = buf.indexOf(0, 17);
-                        streaming.topic = this.dec.decode(buf.subarray(17, i_topic_end));
-                        if (second) {
-                            second = false;
-                            // 是 table
-                            streaming.schema = DdbObj.parse(buf.subarray(i_topic_end + 1), this.le);
-                            console.log('Received streaming table schema:', streaming.schema);
-                            streaming.window = {
-                                offset: 0,
-                                rows: 0,
-                                segments: [],
-                            };
-                            assert(streaming.schema.rows === 0, 'schema.rows === 0');
-                            resolve(streaming.schema);
-                        }
-                        else {
-                            // 是 column 片段组成的 any vector
-                            const data = streaming.data = DdbObj.parse(buf.subarray(i_topic_end + 1), this.le);
-                            let { window: win } = streaming;
-                            win.rows += data.value[0].rows;
-                            win.segments.push(data);
-                            if (win.rows >= winsize * 2 && win.segments.length >= 2) {
-                                let winsize_ = 0;
-                                let i = win.segments.length - 1;
-                                // 往前移动至首个累计 winsize_ 超过 winsize 的位置
-                                for (; winsize_ < winsize; i--)
-                                    winsize_ += win.segments[i].rows;
-                                win.segments = win.segments.slice(i);
-                                win.offset += win.rows - winsize_;
-                                win.rows = winsize_;
-                                console.log('exceeds winsize, shrink window.segments', win);
-                            }
-                            streaming.handler(streaming);
-                        }
-                    }
-                    catch (error) {
-                        reject(error);
-                    }
+        ])).value);
+        this.streaming.window = {
+            offset: 0,
+            rows: 0,
+            segments: [],
+        };
+        // 先准备好收到 websocket message 的 callback
+        this.on_message = ({ data: buffer }) => {
+            let { streaming } = this;
+            try {
+                const dv = new DataView(buffer);
+                const buf = new Uint8Array(buffer);
+                streaming.time = dv.getBigInt64(1, this.le);
+                streaming.id = dv.getBigInt64(9, this.le);
+                const i_topic_end = buf.indexOf(0, 17);
+                streaming.topic = this.dec.decode(buf.subarray(17, i_topic_end));
+                // 是 column 片段组成的 any vector
+                const data = streaming.data = DdbObj.parse(buf.subarray(i_topic_end + 1), this.le);
+                let { window: win } = streaming;
+                win.rows += streaming.rows = data.value[0].rows;
+                win.segments.push(data);
+                if (win.rows >= winsize * 2 && win.segments.length >= 2) {
+                    let winsize_ = 0;
+                    let i = win.segments.length - 1;
+                    // 往前移动至首个累计 winsize_ 超过 winsize 的位置
+                    for (; winsize_ < winsize; i--)
+                        winsize_ += win.segments[i].value[0].rows;
+                    win.segments = win.segments.slice(i);
+                    win.offset += win.rows - winsize_;
+                    win.rows = winsize_;
                 }
-            };
-            // 发送订阅 websocket message
-            this.websocket.send(concat([
-                Uint8Array.of(1, // from = SubscriberFromType::FROM_API
-                1, // type = SubscriberRPCType::RPC_START
-                Number(DDB.le_client)),
-                Uint32Array.of(args.length),
-                ...args.map(obj => obj.pack())
-            ]));
-        });
+            }
+            catch (error) {
+                // 将 error 交给 handler 处理
+                streaming.error = error;
+            }
+            this.streaming.handler(streaming);
+        };
+        return this.streaming;
     }
 }
 export let ddb = window.ddb = new DDB();