nv-buf-storage 1.3.2

package/clses/non-scalable.js

@@ -0,0 +1,28 @@
+const {add_mthds} = require("nv-facutil-simple-patch-cls");
+
+const _fl = require("nv-buf-free-list2");
+const _cmmn_mthds     = require("../mthds/cmmn");
+const _specific_mthds = require("../mthds/non-scalable"); 
+
+function NonScalableStorage(ta_or_freelist,data_u8a) {
+    if(ta_or_freelist.buffer && ta_or_freelist.byteOffset !== undefined && ta_or_freelist.byteLength !== undefined) {
+        const FreeList = _fl.NonScalableFreeList;
+        this._fl   = new FreeList(ta_or_freelist);
+    } else {
+        this._fl   = ta_or_freelist;
+    }
+    this._data = data_u8a;
+    this._dv    = new DataView(this._data.buffer,this._data.byteOffset,this._data.byteLength);
+    Object.defineProperty(this,"_fl",  {enumerable:false});
+    Object.defineProperty(this,"_dv",  {enumerable:false});
+    Object.seal(this);
+}
+add_mthds(NonScalableStorage,_cmmn_mthds);
+add_mthds(NonScalableStorage,_specific_mthds);
+
+Object.defineProperty(NonScalableStorage.prototype,"cond_is_scalable",{get:function(){return false}});
+Object.defineProperty(NonScalableStorage.prototype,"underlying_bytsz",{get:function(){return this._fl.buf.byteLength+this._data.byteLength;}});
+
+module.exports = NonScalableStorage;
+
+

package/clses/scalable.js

@@ -0,0 +1,28 @@
+const {add_mthds} = require("nv-facutil-simple-patch-cls");
+
+const _fl = require("nv-buf-free-list2");
+const _cmmn_mthds     = require("../mthds/cmmn");
+const _specific_mthds = require("../mthds/scalable"); 
+
+function ScalableStorage(ta_or_freelist,data_u8a) {
+    if(ta_or_freelist.buffer && ta_or_freelist.byteOffset !== undefined && ta_or_freelist.byteLength !== undefined) {
+        const FreeList = _fl.ScalableFreeList;
+        this._fl   = new FreeList(ta_or_freelist);
+    } else {
+        this._fl   = ta_or_freelist;
+    }
+    this._data = data_u8a;
+    this._dv    = new DataView(this._data.buffer,this._data.byteOffset,this._data.byteLength);
+    Object.defineProperty(this,"_fl",  {enumerable:false});
+    Object.defineProperty(this,"_dv",  {enumerable:false});
+    Object.seal(this);
+}
+add_mthds(ScalableStorage,_cmmn_mthds);
+add_mthds(ScalableStorage,_specific_mthds);
+
+Object.defineProperty(ScalableStorage.prototype,"cond_is_scalable",{get:function(){return true}});
+Object.defineProperty(ScalableStorage.prototype,"underlying_bytsz",{get:function(){return this._fl.buf.byteLength+this._data.byteLength;}});
+
+module.exports = ScalableStorage;
+
+

package/com.sh

@@ -0,0 +1 @@
+nv_cli_build -g nvbufstorage -m -i index.js -o DIST/nv-buf-storage-bw.js

package/index.d.ts

@@ -0,0 +1,108 @@
+export type Range = [number, number];
+
+export type DebugWriteResult = [string, Range];
+
+export class Buf {
+
+    constructor(init_max_alocable_sz?: number, init_max_nd_cnt?: number);
+
+    dloc(r: Range): void;
+
+    aloc(sz: number): Range;
+
+    algn_aloc(sz: number, algn: number): Range;
+
+    // string
+    rstr(r: Range): string;
+    wstr(s: string): Range;
+
+    // json
+    rjson(r: Range): any;
+    wjson(o: any): Range;
+
+    // object (v8 serializer)
+    ro(r: Range): any;
+    wo(o: any): Range;
+
+    // unsigned
+    ru8(r: Range): number;
+    ru16(r: Range): number;
+    ru32(r: Range): number;
+    ru64(r: Range): bigint;
+
+    // signed
+    ri8(r: Range): number;
+    ri16(r: Range): number;
+    ri32(r: Range): number;
+    ri64(r: Range): bigint;
+
+    // float
+    rf16?(r: Range): number;
+    rf32(r: Range): number;
+    rf64(r: Range): number;
+
+    // write number
+    wu8(n: number): Range;
+    wu16(n: number, algn?: boolean): Range;
+    wu32(n: number, algn?: boolean): Range;
+    wu64(n: bigint | number, algn?: boolean): Range;
+
+    wi8(n: number): Range;
+    wi16(n: number, algn?: boolean): Range;
+    wi32(n: number, algn?: boolean): Range;
+    wi64(n: bigint | number, algn?: boolean): Range;
+
+    wf16?(n: number, algn?: boolean): Range;
+    wf32(n: number, algn?: boolean): Range;
+    wf64(n: number, algn?: boolean): Range;
+
+    // typed arrays read
+    ru8a(r: Range): Uint8Array;
+    ru8ca(r: Range): Uint8ClampedArray;
+    ru16a(r: Range): Uint16Array;
+    ru32a(r: Range): Uint32Array;
+    ru64a(r: Range): BigUint64Array;
+
+    ri8a(r: Range): Int8Array;
+    ri16a(r: Range): Int16Array;
+    ri32a(r: Range): Int32Array;
+    ri64a(r: Range): BigInt64Array;
+
+    rf16a?(r: Range): Float16Array;
+    rf32a(r: Range): Float32Array;
+    rf64a(r: Range): Float64Array;
+
+    rdv(r: Range): DataView;
+
+    // typed arrays write
+    wu8a(a: Uint8Array): Range;
+    wu8ca(a: Uint8ClampedArray): Range;
+    wu16a(a: Uint16Array): Range;
+    wu32a(a: Uint32Array): Range;
+    wu64a(a: BigUint64Array): Range;
+
+    wi8a(a: Int8Array): Range;
+    wi16a(a: Int16Array): Range;
+    wi32a(a: Int32Array): Range;
+    wi64a(a: BigInt64Array): Range;
+
+    wf16a?(a: Float16Array): Range;
+    wf32a(a: Float32Array): Range;
+    wf64a(a: Float64Array): Range;
+
+    wdv(v: DataView): Range;
+
+    // debug
+    debug_read(t: string, r: Range): any;
+    debug_write(o: any): DebugWriteResult;
+}
+
+declare function create(init_max_alocable_sz?: number, init_max_nd_cnt?: number): Buf;
+
+declare namespace create {
+    export { Buf };
+}
+
+export = create;
+
+

package/index.js

@@ -0,0 +1,148 @@
+const {is_overlaped,get_cls_from_dump_fst_byte} = require("nv-facutil-slct-ta");
+const _fl = require("nv-buf-free-list2"); 
+const _util = require("./util");
+
+const  ScalableStorage = require("./clses/scalable");
+const  NonScalableStorage = require("./clses/non-scalable");
+
+
+const {
+    _COND_IS_NODE_ENV,
+    read_sz_from_sz_buf,
+    sync_read_segment_from_file,
+    read_preload_info,
+    read_brief_info_from_dump,
+} = _util;
+
+
+const creat = (
+    init_max_alocable_sz=8192,
+    init_max_nd_cnt=64,
+    scalable = true,
+    fl_underlying =null,
+    data_underlying=null
+) => {
+    var fl   = _fl(init_max_alocable_sz,init_max_nd_cnt,scalable,fl_underlying);
+    var u8a;
+    if(data_underlying === null) {
+        u8a  = new Uint8Array(init_max_alocable_sz);
+    } else {
+        var data_underlying_src  = data_underlying.src;
+        var data_underlying_si   = data_underlying.offset??0;
+        var data_underlying_ei   = data_underlying_si + init_max_alocable_sz;
+        if(is_overlaped(fl.buf,[data_underlying_src,data_underlying_si,data_underlying_ei])) {
+            throw(new Error(`freelist underlying is overlaped with data underlying`));
+        } else {
+            u8a = new Uint8Array(data_underlying_src,data_underlying_si,init_max_alocable_sz);
+        }
+    }
+    const Storage = scalable?ScalableStorage:NonScalableStorage;
+    return new Storage(fl.buf,u8a);
+} 
+
+const load = async (src,data,scalable = true) => {
+    const Storage = scalable?ScalableStorage:NonScalableStorage;
+    if(data !== undefined) {
+        var fl = await _fl.load(src,scalable);
+        if(_COND_IS_NODE_ENV && typeof(data)=== "string") {
+            const _path = require("path");const _fs = require("fs"); const src_fn = _path.resolve(data); const _rfile = require("nv-file-read2").simple;
+            var pair = await _rfile(src_fn,0,Infinity,Infinity,{_fs,_path});
+            if(pair[0]) {
+                data = pair[1];
+            } else      {
+                throw(pair[1]);
+            }
+        } else                                            {
+
+        }
+        //
+        var actual_sz   = data.byteLength;
+        var expected_sz = fl.max_alocable_unit_cnt_v;
+        if(actual_sz < expected_sz) {
+            var ndata = new Uint8Array(expected_sz);
+            ndata.set(data,0);
+            data = ndata;
+        } else {
+        }
+        return new Storage(fl,data);
+    } else {
+        var fl; var datasz;
+        //整体 ["flsz(f64le)","fl(u8a)","datasz(f64le)","data(u8a)"];
+        if(_COND_IS_NODE_ENV && typeof(src)=== "string") {
+            const _path = require("path");const _fs = require("fs"); const src_fn = _path.resolve(src); const _rfile = require("nv-file-read2").simple;
+            var {flsz,max_alocable_unit_cnt_v} = read_preload_info(src_fn,_fs);
+            var dataszu8a =  sync_read_segment_from_file(src_fn,8+flsz,8,_fs);
+            datasz = read_sz_from_sz_buf(dataszu8a);
+            var pair0 = await _rfile(src_fn,8,        8+flsz, 65536,{_fs,_path});
+            var pair1 = await _rfile(src_fn,8+flsz+8,Infinity,65536,{_fs,_path});
+            if(pair0[0] && pair1[0]) {
+                fl      = await _fl.load(pair0[1],scalable);
+                data = pair1[1]; 
+            } else if(pair0[0])      {
+                throw(pair0[1])
+            } else if(pair1[0]){
+                throw(pair1[1])
+            } else {
+                throw([pair0[1],pair1[1]])
+            }
+        } else {
+            var u8a = src;
+            var {flsz,max_alocable_unit_cnt_v}  = read_preload_info(u8a,undefined);  
+            fl      = await _fl.load(u8a.subarray(8,8+flsz),scalable);
+            var dv = new DataView(u8a.buffer);
+            datasz  = dv.getFloat64(8+flsz,true);
+            data = u8a.subarray(8+flsz+8);
+        }
+        ////
+        {
+            var actual_sz   = data.byteLength;
+            var expected_sz = fl.max_alocable_unit_cnt_v;
+            if(actual_sz < expected_sz) {
+                var ndata = new Uint8Array(expected_sz);
+                ndata.set(data,0);
+                data = ndata;
+            } else {
+            }
+            return new Storage(fl,data);
+        }
+    }
+}
+
+
+module.exports = creat;
+module.exports.load = load;
+module.exports.ScalableStorage = ScalableStorage;
+module.exports.NonScalableStorage = NonScalableStorage;
+//
+module.exports._util = _util;
+
+const creat_non_scalable_on_external_ab = (
+    underlying_ab,underlying_offset =0,
+    init_max_alocable_sz=1024*1024*1024*128,
+    init_max_nd_cnt     =2**26,
+)=>{
+    var flsz  =  _fl._type.calc_fl_underlying_bytsz(init_max_alocable_sz,init_max_nd_cnt);
+    return  creat(init_max_alocable_sz,init_max_nd_cnt,false,{src:underlying_ab,offset:underlying_offset},{src:underlying_ab, offset:underlying_offset+flsz});
+}
+
+const load_non_scalable_from_external_ab = (
+    underlying_ab,
+    underlying_offset =0,
+    underlying_sz
+)=>{
+    underlying_sz = underlying_sz??underlying_ab.byteLength-underlying_offset;
+    var u8a     = new Uint8Array(underlying_ab,underlying_offset,underlying_sz);
+    const TaCls = get_cls_from_dump_fst_byte(u8a[0]);
+    var preload = new TaCls(underlying_ab,underlying_offset,6);
+    var datasz  = preload[5];
+    var flsz    = underlying_sz - datasz;
+    var flta    = new TaCls(underlying_ab,underlying_offset,flsz/TaCls.BYTES_PER_ELEMENT);
+    var fl      = new _fl.NonScalableFreeList(flta);
+    var data    = new Uint8Array(underlying_ab,flsz,datasz);
+    return new NonScalableStorage(fl,data);   
+}
+
+module.exports.creat_non_scalable_on_external_ab   = creat_non_scalable_on_external_ab;
+module.exports.load_non_scalable_from_external_ab  = load_non_scalable_from_external_ab;
+
+