node-red-contrib-prib-functions 0.21.0 → 0.23.2

package/README.md

@@ -30,6 +30,8 @@ being outside 3 standard deviations from mean. This can be changed to median and
 A set of data analysis functions that can be run over an array of data
 
 Single value metrics:
+* autocorrelation
+* Autocovariance
 * Average/Mean
 * Maximum
 * Median
@@ -109,53 +111,104 @@ Messages generates a message for each row or record.
 
 Transformations:
 
-*   Array to CSV
-*   Array to HTML
-*   Array to ISO8385
-*   Array to Messages
-*   Array to xlsx / xlsx object (excel uses [xlsx][7])  
+*   Array to
+    * CSV
+    * HTML
+    * ISO8385
+    * Messages
+    * xlsx / xlsx object (excel uses [xlsx][7])  
 *   AVRO to JSON (uses [avsc][6])
-*   Buffer to comprossed 
+*   Buffer to compressed 
 *   Confluence to JSON
-*   Compressed to Buffer
-*   Compressed to String
-*   COmpressed to JSON
-*   CSV to Array
-*   CSV to HTML
-*   CSV to Messages
-*   CSVWithHeader to Array
-*   CSVWithHeader to HTML
-*   CSVWithHeader to JSON
+*   Compressed to
+    * Buffer
+    * String
+    * JSON
+*   CSV to
+    * Array
+    * HTML
+    * Messages
+*   CSVWithHeader to
+    * Array
+    * HTML
+    * JSON
+*   Date to
+    * is between
+    * ISO string 
+    * locale string
+    * range limit
 *   ISO8385 to Array
 *   ISO8385 to JSON
-*   JSON to Array
-*   JSON to Confluence
-*   JSON to CSV
-*   JSON to AVRO (uses [avsc][6])
-*   JSON to ISO8385
-*   JSON to Messages
-*   JSON to String
-*   JSON to xlsx / xlsx object (excel uses [xlsx][7])  
-*   JSON to XML (uses [fast-xml-parser][4])
-*   String to JSON
-*   path to Basename
-*   path to Dirname
-*   path to Extname
-*   path to Format
-*   path to Is Absolute
-*   path to Join
-*   path to Parse
-*   path to Normalize
-*   path to Resolve
+*   JSON to 
+    * Array
+    * Confluence
+    * CSV
+    * AVRO (uses [avsc][6])
+    * ISO8385
+    * Messages
+    * String
+    * xlsx / xlsx object (excel uses [xlsx][7])  
+    * XML (uses [fast-xml-parser][4])
+* Number
+    * is between
+    * range limit
+*   path
+    * Basename
+    * Dirname
+    * Extname
+    * Format
+    * Is Absolute
+    * Join
+    * Parse
+    * Normalize
+    * Resolve
 *   snappy compress (uses [snappy][5], must install separately)
 *   snappy uncompress (uses [snappy][5], must install separately)
+*   String to
+    * Append
+    * Array By Delimiter
+    * At
+    * Camelize
+    * Capitalize
+    * Compressed
+    * Char At
+    * Char Code At"
+    * Code Point At
+    * Concat
+    * Date
+    * Delimiter On Case
+    * _ to space
+    * _ to space Capitilize
+    * Drop square bracket prefix
+    * Ends With
+    * Float
+    * Get Word
+    * Integer
+    * is Between
+    * Lower Case
+    * Number
+    * Prepend
+    * JSON
+    * Range Limit
+    * Split
+    * Starts With
+    * Timestamp
+    * Title
+    * Title Grammatical
+    * Trim
+    * Trim End
+    * Trim Start
+    * Upper Case
+    * Xml String Encode
+
+
 *   xlsx / xlsx object to array/JSON (excel uses [xlsx][7])  
 *   XML to JSON (uses [fast-xml-parser][4])
 
 Note, snappy needs to be installed separately as can have issues with auto install as build binaries.
 
 With xlsx object one can use the function in [xlsx][7] against the object in functions node.
-
+"
 Example AVRO with schema
 
 ![Transform AVRO](documentation/transformArvo.jpg "Transform AVRO example")
@@ -300,6 +353,9 @@ Test/example flow in test/generalTest.json
 ------------------------------------------------------------
 
 # Version
+0.23.0 Removes bug in test, more translation
+
+0.22.0 Add autocovariance + autocorealationship to real time data analystics, improves test
 
 0.21.0 Add lag/seasonal to real time data analystics
 

package/dataAnalysis/arrayAverage.js

@@ -0,0 +1,13 @@
+const arrayTypesForEach=require("./arrayTypesForEach.js")
+require("./arraySum.js")
+
+arrayTypesForEach(object=>{
+	if(object.prototype.average==null)
+		Object.defineProperty(object.prototype, "average", {
+			value(i=0,end=this.length-1) {
+				return this.sum(i,end)/this.length;
+			},
+			writable: true,
+			configurable: true
+		});
+})

package/dataAnalysis/arraySum.js

@@ -1,3 +1,4 @@
+/*
 [Array,Float32Array,Float64Array].forEach(object=>{
 	if(object.prototype.sum==null)
 		Object.defineProperty(object.prototype, "sum", {
@@ -8,4 +9,17 @@
 			writable: true,
 			configurable: true
 		});
+})
+*/
+const arrayTypesForEach=require("./arrayTypesForEach.js")
+arrayTypesForEach(object=>{
+	if(object.prototype.sum==null)
+		Object.defineProperty(object.prototype, "sum", {
+			value(i=0,end=this.length-1,sum=0) {
+				for(;i<=end;i++) sum+=this[i];
+				return sum;
+			},
+			writable: true,
+			configurable: true
+		});
 })

package/dataAnalysis/arrayTypesForEach.js

@@ -0,0 +1,5 @@
+const typedArrays= {Array:Array,Int8Array:Int8Array,Uint8Array:Uint8Array,Uint8ClampedArray:Uint8ClampedArray,Int16Array:Int16Array,Uint16Array:Uint16Array,
+    Int32Array:Int32Array,Uint32Array:Uint32Array,Float32Array:Float32Array,
+    Float64Array:Float64Array,BigInt64Array:BigInt64Array,BigUint64Array:BigUint64Array}
+ 
+module.exports=(call)=>Object.keys(typedArrays).map(t=>typedArrays[t]).forEach(type=>call(type))

package/dataAnalysis/autocorrelation.js

@@ -0,0 +1,29 @@
+
+require("./arrayAverage.js");
+const arrayTypesForEach=require("./arrayTypesForEach.js")
+arrayTypesForEach(object=>{
+    if(object.prototype.autocovariance==null)
+        Object.defineProperty(object.prototype, "autocovariance", {
+            value(lag,avg=this.average()) {
+                let autoCov = 0
+                const vectorSize=this.length
+                const vectorSizeLagged=vectorSize-lag
+                for(let i=0; i<vectorSizeLagged; i++) {
+                    autoCov += ((this[i+lag])-avg)*(this[i]-avg)
+                }
+                return (1/(vectorSize-1))*autoCov
+            },
+            writable: true,
+            configurable: true
+        })
+})
+arrayTypesForEach(object=>{
+    if(object.prototype.autocorrelation==null)
+        Object.defineProperty(object.prototype, "autocorrelation", {
+            value(lag,avg=this.average()) {
+                return this.autocovariance(lag, avg) / this.autocovariance(0, avg)
+            },
+            writable: true,
+            configurable: true
+        })
+})

package/dataAnalysis/dataAnalysis.html

@@ -47,7 +47,7 @@
         		}
            	});
 	       	$("#node-input-action").change(function() {
-				if(["differenceSeasonal","differenceSeasonalSecondOrder","realtime","realtimePredict"].includes( $(this).val() )) {
+				if(["autocovariance","autocorrelation","differenceSeasonal","differenceSeasonalSecondOrder","realtime","realtimePredict"].includes( $(this).val() )) {
         			$(".form-row-http-in-lag").show(); 
         		} else {
         			$(".form-row-http-in-lag").hide();
@@ -158,6 +158,8 @@
     <div class="form-row">
         <label for="node-input-action"><i class="fa fa-list-ul"></i> Method</label>
         <select id="node-input-action" placeholder="action">
+            <option value="autocovariance">Autocovariance</option>
+            <option value="autocorrelation">Autocorrelation</option>
             <option value="avg">Average</option>
             <option value="covariance">Covariance</option>
             <option value="corelationship">Corelationship</option>

package/dataAnalysis/dataAnalysis.js

@@ -1,6 +1,7 @@
 const logger = new (require("node-red-contrib-logger"))("Data Analysis");
 logger.sendInfo("Copyright 2020 Jaroslav Peter Prib");
 
+require("./autocorrelation.js");
 require("./arrayLast");
 require("./arrayDifference")
 require("./arrayDifferenceSeasonal")
@@ -129,6 +130,8 @@ function getColumns(node) {
 	}
 }
 functions={
+	autocovariance:(d,term,node)=>d.autocovariance(node.lag),
+	autocorrelation:(d,term,node)=>d.autocorrelation(node.lag),
 	avg:(d)=>functions.sum(d)/d.length,
 	arrayMax:(d)=>{  // not tested
 		let max=[],indices

package/lib/common.js

@@ -0,0 +1,128 @@
+const error=(node,message,shortMessage,logger)=>{
+	if(logger && logger.active) logger.send({label:"error",node:node.id,error:error,shortMessage});
+	node.error(message);
+	node.status({fill:"red",shape:"ring",text:shortMessage});
+}
+const evalFunction(id,mapping,logger)=>{
+	logger&&logger.sendInfo({label:"evalFunction",id:id,mapping:mapping})
+	try{
+		return eval(mapping);
+	} catch(ex) {
+		logger.sendError({label:"evalFunction error",id:id,mapping:mapping,error:ex.message})
+		throw Error(id+" "+ex.message);
+	}
+}
+const evalInFunction=(node,propertyName)=>{
+	try{
+	    const nodeContext = node.context();
+		const flow = nodeContext.flow;
+		const global = nodeContext.global;
+		const property=node[propertyName];
+		if(property==null) throw Error("no value for "+propertyName);
+		const propertyType=propertyName+"-type";
+		if(! (propertyType in node)) return evalFunction(propertyName,"()=>node."+property) 
+		switch (node[propertyType]){
+		case "num":
+		case "json":
+			return evalFunction(propertyName,"()=>"+property);
+		case "node":
+			return evalFunction(propertyName,"()=>nodeContext.get("+property+")");
+		case "flow":
+			if(flow) throw Error("context store may be memoryonly so flow doesn't work")
+			return evalFunction(propertyName,"()=>flow.get("+property+")");
+		case "global":
+			return evalFunction(propertyName,"()=>global.get("+property+")");
+		case "env":
+			return evalFunction(propertyName,"()=>env.get("+property+")");
+		case "msg":
+			return evalFunction(propertyName,"(msg)=>msg."+property);
+		default:
+			logger.sendInfo({label:"setData unknown type",action:node.action,propertyType:propertyType,type:node[propertyType]});
+			throw Error("unknown type "+node[propertyType])
+		}
+	} catch(ex) {
+		logger.sendError({label:"setup",error:ex.message,stack:ex.stack});
+		throw Error(property+" "+ex.message);
+	}
+}
+
+const argsArray=(node,msg)=>{
+	const args=[];
+	node.argFunction.forEach(callFunction=> {
+		const result=callFunction(msg);
+		args.push(result);
+	});
+	return args;
+}
+const setArgsFunction=(node)=>{
+    node.argFunction=[];
+    node.args.forEach(property=>{
+        try{
+            node.argFunction.push(evalInFunction(node,property).bind(this));
+        } catch(ex) {
+            throw Error("args "+property+" "+ex.message)
+        }
+    })
+}
+
+
+const setTargetFunction=(node)=>{
+    if(node.target) throw Error("target is null")
+    if(node.hasOwnProperty("target-type")) {
+        node.setData=evalFunction("target","data=>data)")
+        return
+    }
+    const nodeContext = node.context()
+    const type=node["target-type"]
+    switch(type){
+        case "node":
+        node.setData=evalFunction("target","data=>nodeContext.set("+node.target+",data)")
+            break
+        case "flow":
+            if(nodeContext.flow) throw Error("context store may be memory only so flow doesn't work")
+            node.setData=evalFunction("target","data=>nodeContext.flow.set("+node.target+",data)")
+            break
+        case "global":
+            node.setData=evalFunction("target","data=>nodeContext.global.set("+node.target+",data)")
+            break
+        case "msg":
+            node.setData=evalFunction("target","(data,msg)=>{msg."+node.target+"=data;}")
+            break
+        default:
+            throw Error("setData unknown type "+type)
+    }
+}
+
+const baseProcess=(msg,call=sourceMatrix[node.action])=>{
+    const value=node.getSource(msg);
+    if(value==null) throw Error("source data not found");
+    const valueObject=(value instanceof Matrix?value:new Matrix(value));
+    return call.apply(valueObject,argsArray(node,msg));
+}
+const baseProcessAndSet=(msg)=>{
+    const result=baseProcess(msg);
+    node.setData.apply(node,[result,msg]);
+}
+function createProcess(msg){
+    const sourceMatrix=new Matrix({rowsMax:node.rows,columns:node.columns,dataType:node.dataType});
+    if(!(node.initialState in sourceMatrix)) throw Error("Invalid initial state "+node.initialState);
+    sourceMatrix[node.initialState]()
+    node.setData.apply(node,[sourceMatrix,msg]);
+}
+function defineProcess(msg){
+    const sourceMatrix=new Matrix({rows:node.rows,columns:node.columns,dataType:node.dataType});
+    node.setData.apply(node,[sourceMatrix,msg]);
+}
+function defineEmptyProcess(msg){
+    if(logger.active) logger.sendInfo({label:"define",arg:{rowsMax:node.row,columns:node.column}});
+    const sourceMatrix=new Matrix({rowsMax:node.rows,columns:node.columns});
+    node.setData.apply(node,[sourceMatrix,msg]);
+}
+function createSize(msg){
+    const sourceMatrix=new Matrix({rows:node.size,columns:node.size});
+    node.setData.apply(node,[sourceMatrix[node.action](),msg]);
+}
+function createDummy(msg){
+    const sourceMatrix=new Matrix(1,1);
+    node.setData.apply(node,[sourceMatrix[node.action].apply(sourceMatrix,argsArray(node,msg)),msg]);
+}