node-red-contrib-prib-functions 0.18.0 → 0.20.4

package/testing/test.js

@@ -31,6 +31,7 @@ function setError(msg,node,err) {
 
 function equalObjects(obj1,obj2,errorFactor) {
 	if( obj1 === obj2 ) return true;
+	if(obj1 instanceof Buffer ) return Buffer.compare(obj1, obj2) === 0
 	if( obj1 === Number.POSITIVE_INFINITY && obj2==="Infinity") return true;
 	if( obj1 === Number.NEGATIVE_INFINITY && obj2==="-Infinity") return true;
 	if( Number.isNaN(obj1) && obj2==="NaN") return true;

package/transform/NumPy.js

@@ -0,0 +1,303 @@
+if(!BigInt.prototype.toJSON)
+    BigInt.prototype.toJSON = function(){return Number(this.toString())}
+if(!Buffer.prototype.toBufferArray)
+    Buffer.prototype.toBufferArray = function() {return this.buffer.slice(this.byteOffset, this.byteOffset + this.byteLength)}
+//const bufferToArrayBuffer = (b) => b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
+const MAGIC_STRING = "\x93NUMPY"
+const defaultVersionStr = "\x01\x00" // version 1.0
+const defaultVersion = 1.0
+const getTotalCells = (shape)=> shape.reduce((a, b) => a * b, 1);
+const dataTypes = {
+    "|b1": {
+        bytes4DataType: 1,
+        name: "bool",
+        cellConstructor: Uint8Array,
+        setDataView: "setUint8"
+      },
+    "<u1": {
+        name: "uint8",
+        bytes4DataType: 8,
+        cellConstructor: Uint8Array,
+        setDataView: "setUint8"
+    },
+    "|u1": {
+        name: "uint8",
+        bytes4DataType: 8,
+        cellConstructor: Uint8Array,
+        setDataView: "setUint8"
+    },
+    "|i1": {
+        name: "int8",
+        bytes4DataType: 8,
+        cellConstructor: Int8Array,
+        setDataView: "setInt8"
+    },
+    "<u2": {
+        name: "uint16",
+        bytes4DataType: 16,
+        cellConstructor: Uint16Array,
+        setDataView: "setUint16"
+    },
+    "<i1": {
+        name: "int8",
+        bytes4DataType: 8,
+        cellConstructor: Int8Array,
+        setDataView: "setInt16"
+    },
+    "<i2": {
+        name: "int16",
+        bytes4DataType: 16,
+        cellConstructor: Int16Array,
+        setDataView: "setInt16"
+    },
+    "<u4": {
+        name: "uint32",
+        bytes4DataType: 32,
+        cellConstructor: Int32Array,
+        setDataView: "setInt32"
+    },
+    "<i4": {
+        name: "int32",
+        bytes4DataType: 32,
+        cellConstructor: Int32Array,
+        setDataView: "setInt32"
+    },
+    "<u8": {
+        name: "uint64",
+        bytes4DataType: 64,
+        cellConstructor: BigUint64Array,
+        setDataView: "setBigUint64"
+    },
+    "<i8": {
+        name: "int64",
+        bytes4DataType: 64,
+        cellConstructor: BigInt64Array,
+        setDataView: "setBigInt64"
+    },
+    "<f4": {
+        name: "float32",
+        bytes4DataType: 32,
+        cellConstructor: Float32Array,
+        setDataView: "setFloat32"
+    },
+    "<f8": {
+        name: "float64",
+        bytes4DataType: 64,
+        cellConstructor: Float64Array,
+        setDataView: "setFloat64"
+    },
+};
+
+/*
+‘S’ - swap dtype from current to opposite endian
+{‘<’, ‘little’} - little endian
+{‘>’, ‘big’} - big endian
+{‘=’, ‘native’} - native order
+{‘|’, ‘I’} - ignore (no change to byte order)
+
+b boolean
+i signed integer
+u unsigned integer f floating-point
+c complex floating-point
+m timedelta
+M datetime
+O object
+S (byte-)string
+U Unicode
+V void
+
+
+* Float data
+    typestr == '>f4'
+    descr == [('','>f4')]
+
+* Complex double
+    typestr == '>c8'
+    descr == [('real','>f4'), ('imag','>f4')]
+
+* RGB Pixel data
+    typestr == '|V3'
+    descr == [('r','|u1'), ('g','|u1'), ('b','|u1')]
+
+* Mixed endian (weird but could happen).
+    typestr == '|V8' (or '>u8')
+    descr == [('big','>i4'), ('little','<i4')]
+
+* Nested structure
+    struct {
+        int ival;
+        struct {
+            unsigned short sval;
+            unsigned char bval;
+            unsigned char cval;
+        } sub;
+    }
+    typestr == '|V8' (or '<u8' if you want)
+    descr == [('ival','<i4'), ('sub', [('sval','<u2'), ('bval','|u1'), ('cval','|u1') ]) ]
+
+* Nested array
+    struct {
+        int ival;
+        double data[16*4];
+    }
+    typestr == '|V516'
+    descr == [('ival','>i4'), ('data','>f8',(16,4))]
+
+* Padded structure
+    struct {
+        int ival;
+        double dval;
+    }
+    typestr == '|V16'
+    descr == [('ival','>i4'),('','|V4'),('dval','>f8')]
+*/
+const dataTypeToNumpyDescr = new Map([
+    ["float32", "<f4"],
+    ["float64", "<f8"],
+    ["int8",  "<i1"],
+    ["int16", "<i2"],
+    ["int32", "<i4"],
+    ["int64", "<i8"],
+    ["uint8",  "<u1"],
+    ["uint16", "<u2"],
+    ["uint32", "<u4"],
+    ["uint64", "<u8"],
+    ["bool", "|b1"],
+  ]);
+/*
+function array2Npy (tensor) {
+    const versionStr = "\x01\x00"; // version 1.0
+    const shapeStr = tensor.shape.join(",") + ","
+    const descr = dataTypeToNumpyDescr.get(tensor.dtype);
+    const header = "{'descr':"+descr+", 'fortran_order': false, 'shape': ("+shapeStr+"), }"
+    const unpaddedLength = MAGIC_STRING.length + versionStr.length + 2 + header.length;
+    // Spaces to 16-bit align.
+    const padding = " ".repeat((16 - (unpaddedLength % 16)) % 16)
+    header += padding;
+    // Number of bytes is in the Numpy descr
+    const bytesPerElement = Number.parseInt(descr[2], 10)
+    const dataLen = bytesPerElement * getTotalCells(tensor.shape)
+    const totalSize = unpaddedLength + padding.length + dataLen
+    const arrayBuffer = new ArrayBuffer(totalSize)
+    const view = new DataView(arrayBuffer)
+    let pos = writeStrToDataView(view, MAGIC_STRING + versionStr, 0)
+    view.setUint16(pos, header.length, true);
+    pos += 2 + writeStrToDataView(view, header, pos);
+    const setDataView = view.setDataView[dataTypes[descr].setDataView];
+    for (let i = 0; i < data.length; i++) {
+         view.setDataView(pos,data[i],true)
+        pos += bytesPerElement;
+    }
+    return arrayBuffer;
+}
+*/
+function writeStrToDataView(dataView, str, pos) {
+    const sl=str.length
+    for (let i = 0; i < sl; i++) {
+        dataView.setInt8(pos + i, str.charCodeAt(i));
+    }
+    return pos + sl;
+  }
+function NumPy (tensor){
+    this.fortran_order=false
+    this.version=defaultVersion
+    return this.parse(tensor)
+}
+NumPy.prototype.getBytes4DataType = function(){
+    const descr  = dataTypeToNumpyDescr.get(this.dtype);
+    return Number.parseInt(descr[2], 10)
+}
+NumPy.prototype.getDesc = function() {
+    return dataTypeToNumpyDescr.get(this.dtype);
+}
+NumPy.prototype.setDType = function(dataType) {
+    this.dataType=dataType
+    const descr=dataTypeToNumpyDescr.get(this.dataType);
+    if(descr==null) throw Error("data type not found for "+dataType)
+    this.setDescr(descr)
+    return this
+}
+NumPy.prototype.setDescr = function(descr) {
+    this.descr=descr
+    const dataTypeDetails = dataTypes[this.descr];
+    this.dataType=dataTypeDetails.name
+    this.setDataView = dataTypeDetails.setDataView;
+    this.cellConstructor = dataTypeDetails.cellConstructor
+    this.bytes4DataType = Number.parseInt(this.descr[2], 10)
+    return this
+}
+NumPy.prototype.toNpy = function () {
+    const versionStr = "\x01\x00"; // version 1.0
+    const shapeStr = this.shape.join(",") + ","
+    let header = "{'descr':"+this.descr+", 'fortran_order': false, 'shape': ("+shapeStr+"), }"
+    const unpaddedLength = MAGIC_STRING.length + versionStr.length + 2 + header.length;
+    // 16-bit align with spaces
+    header += " ".repeat((16 - (unpaddedLength % 16)) % 16);
+    // SIze of Npy
+    const totalSize = header.length + this.bytes4DataType * getTotalCells(this.shape)
+//    const totalSize = unpaddedLength + padding.length + this.bytes4DataType * getTotalCells(this.shape)
+    const arrayBuffer = new ArrayBuffer(totalSize)
+    const view = new DataView(arrayBuffer)
+    let pos = writeStrToDataView(view, MAGIC_STRING + defaultVersionStr, 0)
+    view.setUint16(pos, header.length, true);
+    pos += 2 + writeStrToDataView(view, header, pos);
+    const data=this.dataVector
+//    const setDataView = view[this.setDataView];
+    for (let i = 0; i < data.length; i++) {
+//        setDataView(pos,data[i],true)
+        view[pos]=data[i]
+        pos += this.bytes4DataType;
+    }
+    return arrayBuffer;
+}
+NumPy.prototype.toNpyBuffer = function () {
+    return Buffer.from(this.toNpy())
+}
+NumPy.prototype.parse = function(contentIn={dataType:"int8",
+        shape:[1],
+        dataVector:new int8Array(0)}) {
+    if(!contentIn instanceof Object) throw Error("not an object")
+    if(contentIn.hasOwnProperty("dataVector")) {
+        Object.assign(this,contentIn)
+        if(contentIn.dataType) this.setDType(contentIn.dataType)
+        else if(contentIn.descr) this.setDescr(contentIn.descr)
+        else throw Error("data type not defined")
+        if(this.shape==null) this.shape=[1];
+        return this
+    }
+    return this.parseNpy(contentIn)
+}
+NumPy.prototype.parseNpy = function(npyContentIn) {
+//    this.npyContent=npyContentIn instanceof ArrayBuffer ? npyContentIn : bufferToArrayBuffer(npyContentIn) 
+    this.npyContent=npyContentIn instanceof ArrayBuffer ? npyContentIn : npyContentIn.toBufferArray() 
+    const headerLength = new DataView(this.npyContent.slice(8, 10)).getUint8(0)
+    const offsetBytes = 10 + headerLength;
+    const headerContents = new TextDecoder("utf-8").decode(
+        new Uint8Array(this.npyContent.slice(10, 10 + headerLength))
+    );
+    const header = JSON.parse(
+        headerContents
+            .toLowerCase() // True -> true
+            .replace(/'/g, '"')
+            .replace("(", "[")
+            .replace(/,*\),*/g, "]")
+    );
+    this.setDescr(header.descr);
+    this.shape = header.shape
+    this.dataVector=new this.cellConstructor(this.npyContent, offsetBytes)
+    if(header.fortran_order) this.fortran_order=header.fortran_order
+    return this
+}
+NumPy.prototype.toSerializable = function() {
+    return {
+        dataType:this.dataType,
+        fortran_order: this.fortran_order,
+        shape:this.shape,
+        version:this.version,
+        dataVector:this.dataVector
+    }
+}
+NumPy.prototype.toString = function() {
+    return JSON.stringify(this.toSerializable())
+}
+module.exports=NumPy;

package/transform/transform.html

@@ -30,7 +30,8 @@
 			schemaType: {value:"json"},
 			skipLeading: {value:0},
 			skipTrailing: {value:0},
-			delimiter: {value:",",required:true}
+			delimiter: {value:",",required:true},
+			compressionType: {value:"gzip"}
 		},
 		inputs: 1,
 		outputs: 2,
@@ -43,11 +44,51 @@
 			return this.name ||this.actionSource+"=>"+this.actionTarget|| "Transform";
 		},
 		oneditprepare: function() {
-			const actionTarget=this.actionTarget
+			const actionTarget=this.actionTarget;
+			$("#node-input-actionSource").typedInput({type:"actionSource", types:[{
+				value: "actionSource",
+    			options: [
+				{ value: "Array", label: "Array"},
+				{ value: "AVRO", label: "AVRO"},
+				{ value: "Buffer", label: "Buffer"},
+				{ value: "Compressed", label: "Compressed"},
+				{ value: "Confluence", label: "Confluence"},
+				{ value: "CSVWithHeader", label: "CSV with header"},
+				{ value: "CSV", label: "CSV"},
+				{ value: "ISO8385", label: "ISO 8583"},
+				{ value: "JSON", label: "JSON"},
+				{ value: "npy", label: "npy"},
+				{ value: "NumPyObject", label: "NumPy Obect"},
+				{ value: "String", label: "String"},
+				{ value: "snappy", label: "Snappy"},
+				{ value: "path", label: "Path"},
+				{ value: "XLSX", label: "XLSX (excel)"},
+				{ value: "XLSXObject", label: "XLSX object (excel)"},
+				{ value: "XML", label: "XML"}
+    			]
+			}]});
+			$("#node-input-compressionType").typedInput({type:"compression", types:[{
+			    value: "compression",
+    			options: [
+					{ value: "setGzip", label: "gzip"},
+					{ value: "setGzipSpeed", label: "gzip speed"},
+					{ value: "setGzipCompression", label: "gzip compression"},
+        			{ value: "setZlib", label: "zip"},
+        			{ value: "setZlibSpeed", label: "zip speed"},
+        			{ value: "setZlibCompression", label: "zip compression"},
+        			{ value: "setLzma", label: "Lempel-Ziv-Markov speed"},
+        			{ value: "setLzmaCompression", label: "Lempel-Ziv-Markov compression"},
+        			{ value: "setBrotli", label: "Brotli"},
+        			{ value: "setSnappy", label: "Snappy"},
+        			{ value: "setFlate", label: "Deflate/Inflate"}
+    			]
+			}]});
+
 			$("#node-input-actionSource").change(function() {
 				const actionSource=$(this).val();
 				$(".form-row-http-in-skip").hide();
 				$(".form-row-http-in-schema").hide();
+				$(".form-row-http-in-compressionType").hide();
 				if(!['CSV','CSVWithHeader'].includes(actionSource)) {
 					$(".form-row-http-in-csv").hide();
 				}
@@ -62,10 +103,19 @@
 					options["XLSXObject"]="XLSX object (excel)";
 					break;
 				case 'AVRO':
+				case 'Buffer':
+					options["Compressed"]="Compressed";
+					break;
 				case 'Confluence':
 					options["JSON"]="JSON";
 					$(".form-row-http-in-schema").show();
 					break;
+				case 'Compressed':
+					options["JSON"]="JSON";
+					options["String"]="String";
+					options["Buffer"]="Buffer";
+					$(".form-row-http-in-compressionType").show();
+					break;
 				case 'CSVWithHeader':
 					options["JSON"]="JSON";
 				case 'CSV':
@@ -82,18 +132,28 @@
 				case 'JSON':
 					options["Array"]="Array";
 					options["AVRO"]="AVRO";
+					options["Compressed"]="Compressed";
 					options["Confluence"]="Confluence";
-					options["CSVWithHeader"]="CSV with header";
 					options["CSV"]="CSV";
 					options["ISO8385"]="ISO 8583";
 					options["HTML"]="HTML";
 					options["JSON"]="JSON";
 					options["Messages"]="Messages";
+					options["npy"]="npy";
+					options["NumPyObject"]="NumPy Object";
 					options["String"]="String";
 					options["XLSX"]="XLSX";
 					options["XLSXObject"]="XLSX Object";
 					options["XML"]="XML";
 					break;
+				case 'npy':
+					options["JSON"]="JSON";
+					options["NumPyObject"]="NumPy Object";
+					break;
+				case 'NumPyObject':
+					options["JSON"]="JSON";
+					break;
+				
 				case 'path':
 					options["Basename"]="basename";
 					options["Dirname"]="dirname";
@@ -108,6 +168,7 @@
 					options["Compress"]="Compress";
 					break;
 				case 'String':
+					options["Compressed"]="Compressed";
 					options["JSON"]="JSON";
 					break;
 				case 'XLSX':
@@ -145,11 +206,15 @@
 				if(!['CSV','CSVWithHeader'].includes(actionTarget)) {
 					$(".form-row-http-in-csv").hide();
 				}
+				$(".form-row-http-in-compressionType").hide();
 				switch (actionTarget) {
 				case 'AVRO':
 				case 'Confluence':
 					$(".form-row-http-in-schema").show();
 					break;
+				case 'Compressed':
+					$(".form-row-http-in-compressionType").show();
+					break;
 				case 'CSV':
 					$(".form-row-http-in-csv").show();
 					break;
@@ -187,21 +252,7 @@
 	</div>
 	<div class="form-row">
 		<label for="node-input-actionSource"><i class="fa fa-list-ul"></i> Source Type </label>
-		<select id="node-input-actionSource" placeholder="actionSource">
-			<option value="Array">Array</option>
-			<option value="AVRO">AVRO</option>
-			<option value="Confluence">Confluence</option>
-			<option value="CSVWithHeader">CSV with header</option>
-			<option value="CSV">CSV</option>
-			<option value="ISO8385">ISO 8583</option>
- 			<option value="JSON">JSON</option> 
-			<option value="String">String</option> 
-			<option value="snappy">snappy</option> 
-			<option value="path">Path</option> 
-			<option value="XLSX">XLSX (excel)</option> 
-			<option value="XLSXObject">XLSX object (excel)</option> 
-			<option value="XML">XML</option> 
-		</select>
+		<input type="text" id="node-input-actionSource">
 	</div>
 	<div class="form-row">
 		<label for="node-input-actionTarget"><i class="fa fa-list-ul"></i> Target Type </label>
@@ -227,7 +278,10 @@
 		<input type="text" id="node-input-schema" style="width:70%">
 		<input type="hidden" id="node-input-schemaType">
 	</div>
-
+	<div class="form-row form-row-http-in-compressionType hide">
+		<label for="node-input-compressionType"><i class="fa fa-envelope"></i> <span data-i18n="common.label.compressionType"> Type </span></label>
+		<input type="text" id="node-input-compressionType">
+	</div>
 
 </script>