dolphindb 3.0.22 → 3.0.24

package/index.d.ts

@@ -300,8 +300,8 @@ export declare class DdbObj<TValue extends DdbValue = DdbValue> {
         - 矩阵对应 DdbMatrixData
         - 字典对应普通的 js 对象 Record<string, any>
         - 图对应 DdbChartValue */
-    data<TResult extends any[]>(this: DdbVectorObj): TResult;
-    data<TResult = any>(this: DdbObj): TResult;
+    data<TResult extends any[]>(this: DdbVectorObj, options?: ConvertOptions): TResult;
+    data<TResult = any>(this: DdbObj, options?: ConvertOptions): TResult;
     [inspect.custom](depth: number, _options: InspectOptions, _inspect: any): string;
     inspect_type(): string;
     /** 自动转换 js string, boolean 为 DdbObj */
@@ -343,9 +343,13 @@ export interface InspectOptions extends UtilInspectOptions {
 export declare function format(type: DdbType, value: DdbValue, le: boolean, options?: InspectOptions): string;
 /** 格式化向量、集合中的第 index 项为字符串，空值返回 'null' 字符串  formatted vector, the index-th item in the collection is a string, a null value returns a 'null' string */
 export declare function formati(obj: DdbVectorObj, index: number, options?: InspectOptions): string;
-export declare function convert(type: DdbType, value: DdbValue, le: boolean): string | number | bigint | boolean | Uint8Array | Int32Array | BigInt64Array | string[] | Int8Array | Int16Array | Float32Array | Float64Array | Uint8Array[] | number[] | DdbFunctionDefValue | DdbDurationValue | DdbDecimal32Value | DdbDecimal64Value | DdbDecimal32VectorValue | DdbDecimal64VectorValue | DdbDecimal128VectorValue | BigInt128Array | DdbSymbolExtendedValue | DdbArrayVectorValue | DdbMatrixValue | DdbObj<DdbValue>[] | DdbDurationVectorValue | DdbChartValue;
+export interface ConvertOptions {
+    /** `'string'` blob 类型数据的格式 */
+    blob?: 'string' | 'binary';
+}
+export declare function convert(type: DdbType, value: DdbValue, le: boolean, { blob }?: ConvertOptions): string | number | bigint | boolean | Uint8Array | DdbFunctionDefValue | DdbDurationValue | DdbDecimal32Value | DdbDecimal64Value | DdbDecimal32VectorValue | Int32Array | DdbDecimal64VectorValue | BigInt64Array | DdbDecimal128VectorValue | BigInt128Array | DdbSymbolExtendedValue | string[] | Int8Array | Int16Array | Float32Array | Float64Array | DdbArrayVectorValue | DdbMatrixValue | Uint8Array[] | DdbObj<DdbValue>[] | DdbDurationVectorValue | DdbChartValue | number[];
 /** 转换一个向量到 js 原生数组 */
-export declare function converts(type: DdbType, value: DdbVectorValue, rows: number, le: boolean): any[];
+export declare function converts(type: DdbType, value: DdbVectorValue, rows: number, le: boolean, options?: ConvertOptions): any[];
 export declare class DdbVoid extends DdbObj<undefined> {
     constructor(value?: DdbVoidType);
 }
@@ -489,31 +493,28 @@ export interface StreamingParams {
     };
     handler(message: StreamingMessage): any;
 }
-export interface StreamingMessage extends StreamingParams {
+export interface StreamingMessage<TRows = any> extends StreamingParams {
     /** server 发送消息的时间 (nano seconds since epoch)   The time the server sent the message (nano seconds since epoch)
         std::chrono::system_clock::now().time_since_epoch() / std::chrono::nanoseconds(1) */
     time: bigint;
     /** message id */
     id: bigint;
-    colnames: string[];
-    /** 最新的 server 有可能先推一个 table schema 过来 */
-    schema?: DdbTableObj;
     /** 订阅主题，即一个订阅的名称。  Subscription topic, which is the name of a subscription.
         它是一个字符串，由订阅表所在节点的别名、流数据表名称和订阅任务名称（如果指定了 actionName）组合而成，使用 `/` 分隔
         It is a string consisting of the alias of the node where the subscription table is located, the stream data table name, and the subscription task name (if actionName is specified), separated by `/` */
     topic: string;
     /** 流数据，类型是 any vector, 其中的每一个元素对应被订阅表的一个列 (没有 name)，列 (DdbObj<DdbVectorValue>) 中的内容是新增的数据值
         Stream data, the type is any vector, each element of which corresponds to a column (without name) of the subscribed table, and the content in the column (DdbObj<DdbVectorValue>) is the new data value */
-    data: DdbObj<DdbVectorObj[]>;
-    /** 新增的流数据行数  Number of new streaming data rows */
-    rows: number;
+    obj: DdbObj<DdbVectorObj[]>;
+    /** 流数据 */
+    data: DdbTableData<TRows>;
     window: {
         /** 建立连接开始 offset = 0, 随着 window 的移动逐渐增加  The establishment of the connection starts offset = 0, and gradually increases as the window moves */
         offset: number;
         /** segments 中 segment.row 的总和  sum of segment.row in segments */
         rows: number;
         /** 每次接收到的 data 组成的数组  An array of data received each time */
-        segments: DdbObj<DdbVectorObj[]>[];
+        segments: DdbTableData<TRows>[];
     };
     /** 成功订阅后，后续推送过来的 message 解析错误，则会设置 error 并调用 handler
         After successfully subscribed, if the subsequently pushed message is parsed incorrectly, the error will be set and the handler will be called. */

package/index.js

@@ -3,7 +3,7 @@ import DayjsCustomParseFormat from 'dayjs/plugin/customParseFormat.js';
 dayjs.extend(DayjsCustomParseFormat);
 import ipaddrjs from 'ipaddr.js';
 const { fromByteArray: buf2ipaddr } = ipaddrjs;
-import { concat, assert, inspect, typed_array_to_buffer, connect_websocket, Lock, genid, seq, zip_object, WebSocketOpen, WebSocketClosed, WebSocketClosing, } from 'xshell';
+import { concat, assert, inspect, typed_array_to_buffer, connect_websocket, Lock, genid, seq, zip_object, WebSocketOpen, WebSocketClosed, WebSocketClosing, decode, } from 'xshell';
 import { t } from './i18n/index.js';
 export const nulls = {
     int8: -0x80, // -128
@@ -1122,20 +1122,20 @@ export class DdbObj {
                 throw new Error(t('vector {{type}} 暂不支持序列化', { type: get_type_name(type) }));
         }
     }
-    data() {
+    data(options) {
         const { form, type, value, le, rows, name } = this;
         switch (form) {
             case DdbForm.scalar:
-                return convert(type, value, le);
+                return convert(type, value, le, options);
             case DdbForm.vector:
             case DdbForm.pair:
             case DdbForm.set: {
-                const data = converts(type, value, rows, le);
+                const data = converts(type, value, rows, le, options);
                 return (form === DdbForm.set ? new Set(data) : data);
             }
             case DdbForm.table: {
                 const cols = value;
-                const jscols = cols.map(col => col.data());
+                const jscols = cols.map(col => col.data(options));
                 const keys = cols.map(({ name }) => name);
                 return {
                     name: name || '',
@@ -1146,20 +1146,20 @@ export class DdbObj {
             }
             case DdbForm.dict: {
                 const [keys, values] = value;
-                return zip_object(keys.data(), values.data());
+                return zip_object(keys.data(options), values.data(options));
             }
             case DdbForm.chart:
                 return value;
             case DdbForm.matrix: {
                 const ncolumns = this.cols;
                 const { rows: _rows, cols: _cols, data } = value;
-                const jsdata = converts(type, data, rows * ncolumns, le);
+                const jsdata = converts(type, data, rows * ncolumns, le, options);
                 return {
                     type,
                     nrows: rows,
                     ncolumns,
-                    rows: _rows?.data(),
-                    columns: _cols?.data(),
+                    rows: _rows?.data(options),
+                    columns: _cols?.data(options),
                     data: seq(rows, i => seq(ncolumns, j => jsdata[j * rows + i]))
                 };
             }
@@ -1742,7 +1742,7 @@ export function formati(obj, index, options = {}) {
         }
     }
 }
-export function convert(type, value, le) {
+export function convert(type, value, le, { blob = 'string' } = {}) {
     switch (type) {
         case DdbType.void:
             return value === DdbVoidType.null ? null : undefined;
@@ -1768,8 +1768,9 @@ export function convert(type, value, le) {
         case DdbType.handle:
         case DdbType.datasource:
         case DdbType.resource:
-        case DdbType.blob:
             return value;
+        case DdbType.blob:
+            return blob === 'string' ? decode(value) : value;
         case DdbType.complex:
         case DdbType.point: {
             const [x, y] = value;
@@ -1802,7 +1803,7 @@ export function convert(type, value, le) {
     }
 }
 /** 转换一个向量到 js 原生数组 */
-export function converts(type, value, rows, le) {
+export function converts(type, value, rows, le, options) {
     if (type < 64 || type >= 128)
         switch (type) {
             // 可以直接用下标取值再转换的类型
@@ -1832,7 +1833,7 @@ export function converts(type, value, rows, le) {
             case DdbType.resource:
             case DdbType.functiondef:
             case DdbType.blob:
-                return Array.prototype.map.call(value, (x) => convert(type, x, le));
+                return Array.prototype.map.call(value, (x) => convert(type, x, le, options));
             case DdbType.void:
                 return [];
             case DdbType.symbol_extended: {
@@ -1841,11 +1842,15 @@ export function converts(type, value, rows, le) {
             }
             case DdbType.complex:
             case DdbType.point:
-                return seq(rows, i => [value[2 * i], value[2 * (i + 1)]]);
+                return seq(rows, i => {
+                    const x = value[2 * i];
+                    const y = value[2 * i + 1];
+                    return [x === nulls.double ? null : x, y === nulls.double ? null : y];
+                });
             case DdbType.uuid:
             case DdbType.ipaddr:
             case DdbType.int128:
-                return seq(rows, i => convert(type, value.subarray(16 * i, 16 * (i + 1)), le));
+                return seq(rows, i => convert(type, value.subarray(16 * i, 16 * (i + 1)), le, options));
             case DdbType.decimal32:
             case DdbType.decimal64:
             case DdbType.decimal128: {
@@ -1858,7 +1863,7 @@ export function converts(type, value, rows, le) {
                 });
             }
             case DdbType.any:
-                return value.map(x => x.data());
+                return value.map(x => x.data(options));
             default:
                 throw new Error(String(DdbType[type] || type) + '[]' + t(' 暂时不支持转换为 js 对象'));
         }
@@ -1867,25 +1872,21 @@ export function converts(type, value, rows, le) {
         return value.map(({ lengths, data, rows }) => {
             let acc_len = 0;
             return Array.prototype.map.call(lengths, (length) => {
-                const result = (() => {
-                    switch (type_) {
-                        case DdbType.decimal32:
-                        case DdbType.decimal64:
-                        case DdbType.decimal128:
-                            return converts(type_, { scale: value.scale, data: data.subarray(acc_len, acc_len + length) }, length, le);
-                        case DdbType.complex:
-                        case DdbType.point:
-                            return converts(type_, data.subarray(acc_len, acc_len + 2 * length), length, le);
-                        case DdbType.uuid:
-                        case DdbType.int128:
-                        case DdbType.ipaddr:
-                            return converts(type_, data.subarray(acc_len, acc_len + 16 * length), length, le);
-                        default:
-                            return converts(type_, data.subarray(acc_len, acc_len + length), length, le);
-                    }
-                })();
-                acc_len += length;
-                return result;
+                switch (type_) {
+                    case DdbType.decimal32:
+                    case DdbType.decimal64:
+                    case DdbType.decimal128:
+                        return converts(type_, { scale: value.scale, data: data.subarray(acc_len, acc_len += length) }, length, le, options);
+                    case DdbType.complex:
+                    case DdbType.point:
+                        return converts(type_, data.subarray(acc_len, acc_len += 2 * length), length, le, options);
+                    case DdbType.uuid:
+                    case DdbType.int128:
+                    case DdbType.ipaddr:
+                        return converts(type_, data.subarray(acc_len, acc_len += 16 * length), length, le, options);
+                    default:
+                        return converts(type_, data.subarray(acc_len, acc_len += length), length, le, options);
+                }
             });
         }).flat();
     }
@@ -3042,8 +3043,9 @@ export class DDB {
             rows: 0,
             segments: [],
         };
-        let schema;
-        const { value: colnames } = await this.call('publishTable', [
+        // 流表推送过来的第一条数据是 schema，需要特殊处理
+        let first = true;
+        const { value: columns } = await this.call('publishTable', [
             'localhost',
             new DdbInt(0),
             this.streaming.table,
@@ -3057,41 +3059,56 @@ export class DDB {
             // 先准备好收到 websocket message 的 callback
             on_more_messages: buffer => {
                 try {
+                    let data;
                     const dv = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
                     const i_topic_end = buffer.indexOf(0, 17);
                     // 首个 message 一定是 table schema, 后续消息是 column 片段组成的 any vector
-                    const data = DdbObj.parse(buffer.subarray(i_topic_end + 1), this.le);
+                    const obj = DdbObj.parse(buffer.subarray(i_topic_end + 1), this.le);
                     // server 推数据时遇到错误会返回 string，一般格式为 error.xxx: 错误信息
-                    if (data.form === DdbForm.scalar && data.type === DdbType.string) {
+                    if (obj.form === DdbForm.scalar && obj.type === DdbType.string) {
                         this.disconnect();
-                        const value = data.value;
+                        const value = obj.value;
                         throw new Error(value.slice(value.indexOf(':') + 1).trim());
                     }
-                    if (!schema) {
-                        schema = data;
-                        return;
+                    if (first) {
+                        data = obj.data();
+                        data.name ||= this.streaming.table;
+                        first = false;
                     }
-                    const { rows } = data.value[0];
-                    win.rows += rows;
-                    win.segments.push(data);
-                    if (win.rows >= winsize * 2 && win.segments.length >= 2) {
-                        let winsize_ = 0;
-                        let i = win.segments.length - 1;
-                        // 往前移动至首个累计 winsize_ 超过 winsize 的位置
-                        for (; winsize_ < winsize; i--)
-                            winsize_ += win.segments[i].value[0].rows;
-                        win.segments = win.segments.slice(i);
-                        win.offset += win.rows - winsize_;
-                        win.rows = winsize_;
+                    else {
+                        const { rows } = obj.value[0];
+                        const types = [];
+                        const js_cols = [];
+                        // 遍历每一列
+                        obj.value.forEach(({ type, value }) => {
+                            types.push(type);
+                            js_cols.push(converts(type, value, rows, obj.le));
+                        });
+                        data = {
+                            name: this.streaming.table || '',
+                            columns,
+                            types,
+                            data: seq(rows, i => zip_object(columns, seq(columns.length, j => js_cols[j][i])))
+                        };
+                        win.rows += rows;
+                        win.segments.push(data);
+                        if (win.rows >= winsize * 2 && win.segments.length >= 2) {
+                            let winsize_ = 0;
+                            let i = win.segments.length - 1;
+                            // 往前移动至首个累计 winsize_ 超过 winsize 的位置
+                            for (; winsize_ < winsize; i--)
+                                winsize_ += win.segments[i].data.length;
+                            win.segments = win.segments.slice(i);
+                            win.offset += win.rows - winsize_;
+                            win.rows = winsize_;
+                        }
                     }
                     this.streaming.handler({
                         ...this.streaming,
                         id: dv.getBigInt64(9, this.le),
                         time: dv.getBigInt64(1, this.le),
-                        rows,
                         topic: this.dec.decode(buffer.subarray(17, i_topic_end)),
-                        colnames,
-                        schema,
+                        obj,
                         data,
                         window: win,
                     });
@@ -3102,9 +3119,9 @@ export class DDB {
                 }
             }
         });
-        console.log(t('订阅流表成功') + ', colnames:', 
+        console.log(t('订阅流表成功') + ', columns:', 
         // string[3](['time', 'stock', 'price'])
-        colnames);
+        columns);
     }
 }
 export class BigInt128Array {