node-red-contrib-prib-functions 0.19.2 → 0.20.4

package/test/dataAnalysisSVD.js

@@ -0,0 +1,31 @@
+const assert=require('assert');
+const should=require("should");
+const svd=require("../dataAnalysis/svd.js");
+require("../dataAnalysis/arrayCompareToPrecision.js");
+
+const expected={
+	orthonormalizedColumns: [
+	  [ -0.3913289321676804, 0.3487948014410755, 0.10448989927485411 ],
+	  [ -0.4639397307294648, -0.34948424610207607, 0.7962721013609022 ],
+	  [ -0.6197155556759748, -0.5188940242462491, -0.5888137410546312 ],
+	  [ -0.4975683100363125, 0.6978194177691509, -0.0912742016622088 ]
+	],
+	singularValues: [ 223.85445199527607, 13.695880509722402, 17.366841363876695 ],
+	orthogonalMatrix: [
+	  [ -0.5061583666044986, -0.7405855572048029, -0.44196916224541927 ],
+	  [ -0.5596333427471312, -0.10788454127728717, 0.8216881692217926 ],
+	  [ -0.6562120309792694, 0.6632450212500965, -0.35984970777398323 ]
+	]
+  }
+
+describe('SVD', function(){
+	it("getEigenVectors", function(done){
+		const data = [[40,50,60],[50,70,60],[80,70,90],[50,60,80]];
+		const vectors = svd(data);
+		console.log(vectors)
+		vectors.orthonormalizedColumns.compareToPrecision(expected.orthonormalizedColumns)
+		vectors.singularValues.compareToPrecision(expected.singularValues)
+		vectors.orthogonalMatrix.compareToPrecision(expected.orthogonalMatrix)
+		done();
+	});
+});	

package/test/euclideanDistance.js

@@ -1,8 +1,8 @@
-const assert=require('assert');
+onst assert=require('assert');
 const should=require("should");
 const ed=require("../dataAnalysis/euclideanDistance.js");
 require("../dataAnalysis/forNestedEach");
-
+c
 describe('euclideanDistance', function() { 
 	it("array forNestedEach", function(done) {
 		const atest=[1,2,3,4];

package/test/transformNumPy.js

@@ -0,0 +1,132 @@
+const assert=require('assert')
+const helper = require("node-red-node-test-helper")
+const NumPy = require("../transform/NumPy.js")
+const transformNode = require("../transform/transform.js");
+const fs = require('fs')
+const path = require('path')
+const filePath = path.join(__dirname, "data/float32vector10.npy")
+const npyFloat32V10 = fs.readFileSync(filePath)
+const npyInt2matrix2x3 = fs.readFileSync(path.join(__dirname, "data/int2matrix2x3.npy"))
+const int2matrix2x3={"dataType":"int64","fortran_order":false,"shape":[1,2,3],"version":1.0,"dataVector":new BigInt64Array([1n,2n,3n,4n,5n,6n])}
+if(!Buffer.prototype.toBufferArray)
+    Buffer.prototype.toBufferArray = function() {return this.buffer.slice(this.byteOffset, this.byteOffset + this.byteLength)}
+
+const float32V10 = new Float32Array([
+	86,  46,  10, 148, 133,
+	86, 103, 118,  62,  49
+  ])
+const NumPyJSON={
+	"dataType":"float32",
+	"fortran_order":false,
+	version:1.0,
+	"shape":[10],
+	"dataVector":{"0":86,"1":46,"2":10,"3":148,"4":133,"5":86,"6":103,"7":118,"8":62,"9":49}
+}
+helper.init(require.resolve('node-red'));
+
+const npy2JSONNode={
+	id : "npy2JSON",
+	type : "transform",
+	name : "npy to JSON",
+	actionSource: "npy",
+	actionTarget: "JSON",
+	sourceProperty:"msg.payload",
+	targetProperty:"msg.payload",
+	topicProperty:"'test topic'"
+};
+const JSON2npyNode={
+	id : "npy2JSON",
+	type : "transform",
+	name : "JSON to npy",
+	actionSource: "JSON",
+	actionTarget: "npy",
+	sourceProperty:"msg.payload",
+	targetProperty:"msg.payload",
+	topicProperty:"'test topic'"
+};
+function getAndTestNodeProperties(o) {
+	const n = helper.getNode(o.id);
+	for(let p in o) n.should.have.property(p, o[p]);
+	return n;
+}
+function testFlow(done,node,data,result) {
+	const flow = [
+		Object.assign(node,{wires : [ [ "outHelper" ],["errorHelper"] ]}),
+		{id :"outHelper",	type : "helper"},
+		{id :"errorHelper",	type : "helper"}
+	];
+	helper.load(transformNode, flow, function() {
+		const n=getAndTestNodeProperties(node);
+		const outHelper = helper.getNode("outHelper");
+		const errorHelper = helper.getNode("errorHelper");
+		outHelper.on("input", function(msg) {
+			console.log("outHelper "+JSON.stringify(msg.payload));
+			if(JSON.stringify(msg.payload)==JSON.stringify(result)) {
+				done();
+			} else {
+				console.log("mismatch  expected: "+JSON.stringify(result) +" returned: "+JSON.stringify(msg.payload));
+				done("mismatch");
+			}
+		});
+		errorHelper.on("input", function(msg) {
+			console.log("errorHelper "+JSON.stringify(msg));
+			done("error  check log output");
+		});
+		n.receive({
+			topic:"test",
+			payload : data
+		});
+	});
+}
+
+describe('NumPy', function() {
+	it("parse npyFloat10",(done)=>{
+		const tensor=new NumPy(npyFloat32V10)
+		assert.deepStrictEqual(tensor.dataType,'float32')
+		assert.deepStrictEqual(tensor.shape,[10])
+		assert.deepStrictEqual(tensor.fortran_order,false)
+		assert.deepStrictEqual(tensor.dataVector,float32V10)
+		done()
+	});
+	it("parse npyInt2matrix2x3",(done)=>{
+		const tensor=new NumPy(npyInt2matrix2x3)
+		assert.deepStrictEqual(tensor.toSerializable(),int2matrix2x3)
+		done()
+	});
+	it("parse int2matrix2x3",(done)=>{
+		const tensor=new NumPy(int2matrix2x3)
+		assert.deepStrictEqual(tensor.toSerializable(),int2matrix2x3)
+		console.log({label:"toString",result:tensor.toString()})
+		const npy=tensor.toNpy()
+		done();
+	});
+/*
+	it("toNpy float32V10",(done)=>{
+		const tensor=new NumPy(NumPyJSON)
+		assert.deepStrictEqual(tensor.toNpy(),npyFloat32V10.toBufferArray())
+		done();
+	});
+
+	it("toNpy int2matrix2x3",(done)=>{
+		const tensor=new NumPy(int2matrix2x3)
+		assert.deepStrictEqual(tensor.toNpy(),npyInt2matrix2x3.toBufferArray())
+		done();
+	});
+*/
+});
+
+describe('transform numpy', function() {
+	beforeEach(function(done) {
+		helper.startServer(done);
+	});
+	afterEach(function(done) {
+		helper.unload();
+		helper.stopServer(done);
+	});
+	it('npy to Array', function(done) {
+		testFlow(done,JSON2npyNode,int2matrix2x3,npyInt2matrix2x3);
+	});
+	it('npy to JSON', function(done) {
+		testFlow(done,npy2JSONNode,npyInt2matrix2x3,int2matrix2x3);
+	});
+});

package/transform/NumPy.js

@@ -0,0 +1,303 @@
+if(!BigInt.prototype.toJSON)
+    BigInt.prototype.toJSON = function(){return Number(this.toString())}
+if(!Buffer.prototype.toBufferArray)
+    Buffer.prototype.toBufferArray = function() {return this.buffer.slice(this.byteOffset, this.byteOffset + this.byteLength)}
+//const bufferToArrayBuffer = (b) => b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
+const MAGIC_STRING = "\x93NUMPY"
+const defaultVersionStr = "\x01\x00" // version 1.0
+const defaultVersion = 1.0
+const getTotalCells = (shape)=> shape.reduce((a, b) => a * b, 1);
+const dataTypes = {
+    "|b1": {
+        bytes4DataType: 1,
+        name: "bool",
+        cellConstructor: Uint8Array,
+        setDataView: "setUint8"
+      },
+    "<u1": {
+        name: "uint8",
+        bytes4DataType: 8,
+        cellConstructor: Uint8Array,
+        setDataView: "setUint8"
+    },
+    "|u1": {
+        name: "uint8",
+        bytes4DataType: 8,
+        cellConstructor: Uint8Array,
+        setDataView: "setUint8"
+    },
+    "|i1": {
+        name: "int8",
+        bytes4DataType: 8,
+        cellConstructor: Int8Array,
+        setDataView: "setInt8"
+    },
+    "<u2": {
+        name: "uint16",
+        bytes4DataType: 16,
+        cellConstructor: Uint16Array,
+        setDataView: "setUint16"
+    },
+    "<i1": {
+        name: "int8",
+        bytes4DataType: 8,
+        cellConstructor: Int8Array,
+        setDataView: "setInt16"
+    },
+    "<i2": {
+        name: "int16",
+        bytes4DataType: 16,
+        cellConstructor: Int16Array,
+        setDataView: "setInt16"
+    },
+    "<u4": {
+        name: "uint32",
+        bytes4DataType: 32,
+        cellConstructor: Int32Array,
+        setDataView: "setInt32"
+    },
+    "<i4": {
+        name: "int32",
+        bytes4DataType: 32,
+        cellConstructor: Int32Array,
+        setDataView: "setInt32"
+    },
+    "<u8": {
+        name: "uint64",
+        bytes4DataType: 64,
+        cellConstructor: BigUint64Array,
+        setDataView: "setBigUint64"
+    },
+    "<i8": {
+        name: "int64",
+        bytes4DataType: 64,
+        cellConstructor: BigInt64Array,
+        setDataView: "setBigInt64"
+    },
+    "<f4": {
+        name: "float32",
+        bytes4DataType: 32,
+        cellConstructor: Float32Array,
+        setDataView: "setFloat32"
+    },
+    "<f8": {
+        name: "float64",
+        bytes4DataType: 64,
+        cellConstructor: Float64Array,
+        setDataView: "setFloat64"
+    },
+};
+
+/*
+‘S’ - swap dtype from current to opposite endian
+{‘<’, ‘little’} - little endian
+{‘>’, ‘big’} - big endian
+{‘=’, ‘native’} - native order
+{‘|’, ‘I’} - ignore (no change to byte order)
+
+b boolean
+i signed integer
+u unsigned integer f floating-point
+c complex floating-point
+m timedelta
+M datetime
+O object
+S (byte-)string
+U Unicode
+V void
+
+
+* Float data
+    typestr == '>f4'
+    descr == [('','>f4')]
+
+* Complex double
+    typestr == '>c8'
+    descr == [('real','>f4'), ('imag','>f4')]
+
+* RGB Pixel data
+    typestr == '|V3'
+    descr == [('r','|u1'), ('g','|u1'), ('b','|u1')]
+
+* Mixed endian (weird but could happen).
+    typestr == '|V8' (or '>u8')
+    descr == [('big','>i4'), ('little','<i4')]
+
+* Nested structure
+    struct {
+        int ival;
+        struct {
+            unsigned short sval;
+            unsigned char bval;
+            unsigned char cval;
+        } sub;
+    }
+    typestr == '|V8' (or '<u8' if you want)
+    descr == [('ival','<i4'), ('sub', [('sval','<u2'), ('bval','|u1'), ('cval','|u1') ]) ]
+
+* Nested array
+    struct {
+        int ival;
+        double data[16*4];
+    }
+    typestr == '|V516'
+    descr == [('ival','>i4'), ('data','>f8',(16,4))]
+
+* Padded structure
+    struct {
+        int ival;
+        double dval;
+    }
+    typestr == '|V16'
+    descr == [('ival','>i4'),('','|V4'),('dval','>f8')]
+*/
+const dataTypeToNumpyDescr = new Map([
+    ["float32", "<f4"],
+    ["float64", "<f8"],
+    ["int8",  "<i1"],
+    ["int16", "<i2"],
+    ["int32", "<i4"],
+    ["int64", "<i8"],
+    ["uint8",  "<u1"],
+    ["uint16", "<u2"],
+    ["uint32", "<u4"],
+    ["uint64", "<u8"],
+    ["bool", "|b1"],
+  ]);
+/*
+function array2Npy (tensor) {
+    const versionStr = "\x01\x00"; // version 1.0
+    const shapeStr = tensor.shape.join(",") + ","
+    const descr = dataTypeToNumpyDescr.get(tensor.dtype);
+    const header = "{'descr':"+descr+", 'fortran_order': false, 'shape': ("+shapeStr+"), }"
+    const unpaddedLength = MAGIC_STRING.length + versionStr.length + 2 + header.length;
+    // Spaces to 16-bit align.
+    const padding = " ".repeat((16 - (unpaddedLength % 16)) % 16)
+    header += padding;
+    // Number of bytes is in the Numpy descr
+    const bytesPerElement = Number.parseInt(descr[2], 10)
+    const dataLen = bytesPerElement * getTotalCells(tensor.shape)
+    const totalSize = unpaddedLength + padding.length + dataLen
+    const arrayBuffer = new ArrayBuffer(totalSize)
+    const view = new DataView(arrayBuffer)
+    let pos = writeStrToDataView(view, MAGIC_STRING + versionStr, 0)
+    view.setUint16(pos, header.length, true);
+    pos += 2 + writeStrToDataView(view, header, pos);
+    const setDataView = view.setDataView[dataTypes[descr].setDataView];
+    for (let i = 0; i < data.length; i++) {
+         view.setDataView(pos,data[i],true)
+        pos += bytesPerElement;
+    }
+    return arrayBuffer;
+}
+*/
+function writeStrToDataView(dataView, str, pos) {
+    const sl=str.length
+    for (let i = 0; i < sl; i++) {
+        dataView.setInt8(pos + i, str.charCodeAt(i));
+    }
+    return pos + sl;
+  }
+function NumPy (tensor){
+    this.fortran_order=false
+    this.version=defaultVersion
+    return this.parse(tensor)
+}
+NumPy.prototype.getBytes4DataType = function(){
+    const descr  = dataTypeToNumpyDescr.get(this.dtype);
+    return Number.parseInt(descr[2], 10)
+}
+NumPy.prototype.getDesc = function() {
+    return dataTypeToNumpyDescr.get(this.dtype);
+}
+NumPy.prototype.setDType = function(dataType) {
+    this.dataType=dataType
+    const descr=dataTypeToNumpyDescr.get(this.dataType);
+    if(descr==null) throw Error("data type not found for "+dataType)
+    this.setDescr(descr)
+    return this
+}
+NumPy.prototype.setDescr = function(descr) {
+    this.descr=descr
+    const dataTypeDetails = dataTypes[this.descr];
+    this.dataType=dataTypeDetails.name
+    this.setDataView = dataTypeDetails.setDataView;
+    this.cellConstructor = dataTypeDetails.cellConstructor
+    this.bytes4DataType = Number.parseInt(this.descr[2], 10)
+    return this
+}
+NumPy.prototype.toNpy = function () {
+    const versionStr = "\x01\x00"; // version 1.0
+    const shapeStr = this.shape.join(",") + ","
+    let header = "{'descr':"+this.descr+", 'fortran_order': false, 'shape': ("+shapeStr+"), }"
+    const unpaddedLength = MAGIC_STRING.length + versionStr.length + 2 + header.length;
+    // 16-bit align with spaces
+    header += " ".repeat((16 - (unpaddedLength % 16)) % 16);
+    // SIze of Npy
+    const totalSize = header.length + this.bytes4DataType * getTotalCells(this.shape)
+//    const totalSize = unpaddedLength + padding.length + this.bytes4DataType * getTotalCells(this.shape)
+    const arrayBuffer = new ArrayBuffer(totalSize)
+    const view = new DataView(arrayBuffer)
+    let pos = writeStrToDataView(view, MAGIC_STRING + defaultVersionStr, 0)
+    view.setUint16(pos, header.length, true);
+    pos += 2 + writeStrToDataView(view, header, pos);
+    const data=this.dataVector
+//    const setDataView = view[this.setDataView];
+    for (let i = 0; i < data.length; i++) {
+//        setDataView(pos,data[i],true)
+        view[pos]=data[i]
+        pos += this.bytes4DataType;
+    }
+    return arrayBuffer;
+}
+NumPy.prototype.toNpyBuffer = function () {
+    return Buffer.from(this.toNpy())
+}
+NumPy.prototype.parse = function(contentIn={dataType:"int8",
+        shape:[1],
+        dataVector:new int8Array(0)}) {
+    if(!contentIn instanceof Object) throw Error("not an object")
+    if(contentIn.hasOwnProperty("dataVector")) {
+        Object.assign(this,contentIn)
+        if(contentIn.dataType) this.setDType(contentIn.dataType)
+        else if(contentIn.descr) this.setDescr(contentIn.descr)
+        else throw Error("data type not defined")
+        if(this.shape==null) this.shape=[1];
+        return this
+    }
+    return this.parseNpy(contentIn)
+}
+NumPy.prototype.parseNpy = function(npyContentIn) {
+//    this.npyContent=npyContentIn instanceof ArrayBuffer ? npyContentIn : bufferToArrayBuffer(npyContentIn) 
+    this.npyContent=npyContentIn instanceof ArrayBuffer ? npyContentIn : npyContentIn.toBufferArray() 
+    const headerLength = new DataView(this.npyContent.slice(8, 10)).getUint8(0)
+    const offsetBytes = 10 + headerLength;
+    const headerContents = new TextDecoder("utf-8").decode(
+        new Uint8Array(this.npyContent.slice(10, 10 + headerLength))
+    );
+    const header = JSON.parse(
+        headerContents
+            .toLowerCase() // True -> true
+            .replace(/'/g, '"')
+            .replace("(", "[")
+            .replace(/,*\),*/g, "]")
+    );
+    this.setDescr(header.descr);
+    this.shape = header.shape
+    this.dataVector=new this.cellConstructor(this.npyContent, offsetBytes)
+    if(header.fortran_order) this.fortran_order=header.fortran_order
+    return this
+}
+NumPy.prototype.toSerializable = function() {
+    return {
+        dataType:this.dataType,
+        fortran_order: this.fortran_order,
+        shape:this.shape,
+        version:this.version,
+        dataVector:this.dataVector
+    }
+}
+NumPy.prototype.toString = function() {
+    return JSON.stringify(this.toSerializable())
+}
+module.exports=NumPy;

package/transform/transform.html

@@ -57,6 +57,8 @@
 				{ value: "CSV", label: "CSV"},
 				{ value: "ISO8385", label: "ISO 8583"},
 				{ value: "JSON", label: "JSON"},
+				{ value: "npy", label: "npy"},
+				{ value: "NumPyObject", label: "NumPy Obect"},
 				{ value: "String", label: "String"},
 				{ value: "snappy", label: "Snappy"},
 				{ value: "path", label: "Path"},
@@ -137,11 +139,21 @@
 					options["HTML"]="HTML";
 					options["JSON"]="JSON";
 					options["Messages"]="Messages";
+					options["npy"]="npy";
+					options["NumPyObject"]="NumPy Object";
 					options["String"]="String";
 					options["XLSX"]="XLSX";
 					options["XLSXObject"]="XLSX Object";
 					options["XML"]="XML";
 					break;
+				case 'npy':
+					options["JSON"]="JSON";
+					options["NumPyObject"]="NumPy Object";
+					break;
+				case 'NumPyObject':
+					options["JSON"]="JSON";
+					break;
+				
 				case 'path':
 					options["Basename"]="basename";
 					options["Dirname"]="dirname";

package/transform/transform.js

@@ -1,7 +1,7 @@
 const logger = new (require("node-red-contrib-logger"))("transform");
 logger.sendInfo("Copyright 2020 Jaroslav Peter Prib");
 const CompressionTool = require('compressiontool');
-
+const NumPy = require("./NumPy.js")
 const regexCSV=/,(?=(?:(?:[^"]*"){2})*[^"]*$)/,
 	Buffer=require('buffer').Buffer,
 	os=require('os'),
@@ -332,6 +332,7 @@ const functions={
 		}
 		return data;
 	},
+	JSONToAVRO: (RED,node,msg,data)=>node.avroTransformer.toBuffer(data), // Encoded buffer.
 	JSONToCompressed: (RED,node,msg,data)=>compressor.compress(JSON.stringify(data),
 		(compressed)=>{
 			node.setData(RED,node,msg,compressed);
@@ -343,7 +344,6 @@ const functions={
 	),
 	JSONToConfluence:JSONToConfluence,
 	JSONToCSV: (RED,node,msg,data)=>Array2csv(node,data),
-	JSONToAVRO: (RED,node,msg,data)=>node.avroTransformer.toBuffer(data), // Encoded buffer.
 	JSONToHTML: (RED,node,msg,data,level=0)=>{
 		if(Array.isArray(data)) {
 			return data.length?"<table><tr>"+data.map((r)=>functions.JSONToHTML(RED,node,msg,r,++level)).join("</tr><tr>")+"</tr><table>":"";
@@ -375,10 +375,15 @@ const functions={
 			node.send(newMsg);
 		}
 	},
+	JSONTonpy: (RED,node,msg,data)=>new NumPy(data).toNpyBuffer(),
+	JSONToNumPyObject: (RED,node,msg,data)=>new NumPy(data),
 	JSONToString: (RED,node,msg,data)=>JSON.stringify(data),
 	JSONToXLSX:JSONToXLSX,
 	JSONToXLSXObject:JSONToXLSXObject,
 	JSONToXML: (RED,node,msg,data)=>json2xmlParser.parse(data),
+	npyToJSON: (RED,node,msg,data)=>new NumPy(data).toSerializable(),
+	npyToNumPyObject: (RED,node,msg,data)=>new NumPy(data),
+	NumPyObjectToJSON: (RED,node,msg,data)=> data.toSerializable(),
 	StringToCompressed: (RED,node,msg,data)=>compressor.compress(data,
 		(compressed)=>{
 			node.setData(RED,node,msg,compressed);
@@ -441,6 +446,23 @@ function evalFunction(id,mapping){
 function is(node,value){
 	return node.actionSource==value||node.actionTarget==value;
 }
+let jsonata;
+function JSONataTransform(data,ok,error){
+/*
+(async () => {
+    return result = await expression.evaluate(data);
+})()
+*/
+
+	this.transformFuncionCompiled.evalFunction(data,{},(error, result) => {
+		if(error) {
+		  console.error(error);
+		  return;
+		}
+		console.log("Finished with", result);
+	});
+}
+
 module.exports = function (RED) {
 	function transformNode(n) {
 		RED.nodes.createNode(this,n);
@@ -513,6 +535,16 @@ module.exports = function (RED) {
 				}
 			}
 		}
+		if(node.transformFuncion && ( node.actionSource=="JSON" || node.actionTarget=="JSON" )) {
+			try{
+				if(!jsonata) jsonata=require('jsonata')
+				node.transformFuncionCompiled = jsonata(node.transformFuncion);
+			} catch (ex) {
+				error(node,ex,"Transform function error");
+				return;
+			}
+		}
+
 		const typeValidate="invalid"+node.actionSource;
 		node.invalidSourceType=typeValidate in functions &! ["XLSX","XLSXObject"].includes(node.actionTarget)?functions[typeValidate]:(()=>false);
 		try {