node-red-contrib-prib-functions 0.16.0 → 0.18.0

package/.vscode/extensions.json

@@ -0,0 +1,6 @@
+{
+    "recommendations": [
+        "bibhasdn.git-easy",
+        "github.vscode-pull-request-github"
+    ]
+}

package/README.md

@@ -3,6 +3,7 @@
 [Node-Red][1] nodes for various functions:
 
 *   Data Analysis  - statistical metrics that has real time option
+*   Matrix
 *	Transform
 *	Test
 *   Load Injector
@@ -50,6 +51,11 @@ Array metrics:
 * Normalise
 * Standardization (Z-score Normalization)
 
+Array data
+* distances 
+* minimum distance(s) between points
+* maximum distance(s) between points
+
 ![Data Analysis](documentation/DataAnalysis.JPG "Data Analysis")
 
 example:
@@ -61,6 +67,36 @@ example:
 
 ![Data Analysis example](documentation/DataAnalysisTest.JPG "Data Analysis example")
 
+
+------------------------------------------------------------
+
+## Matrix
+
+Define a matrix and perform various functions
+
+ * Define / Define Empty / Create / Create Like/ clone"
+ * Add / Add Row to Row / Add to Cell / Add Row / Subtract Cell
+ * Multiple / Multiple Cell / Divide Cell / Divide Row
+ * Transpose
+ * Adjoint
+ * Cofactor
+ * Complement Minor
+ * Identity
+ * Inverse
+ * Determinant
+ * Backward Substitution
+ * Forward Elimination
+ * Gaussian Elimination
+ * Reduced Row EchelonForm
+ * Row Echelon Form
+ * Nearly Equals / Is Square / Get Cell
+ * Sum Row
+ * Swap Rows
+ * To Array Object
+
+
+![Matrix](documentation/matrix.jpg "Matrix")
+
 ------------------------------------------------------------
 
 ## Transform
@@ -115,7 +151,7 @@ With xlsx object one can use the function in [xlsx][7] against the object in fun
 
 Example AVRO with schema
 
-![Transform AVRO](documentation/transformArvo.JPG "Transform AVRO example")
+![Transform AVRO](documentation/transformArvo.jpg "Transform AVRO example")
 
 For Confluence schema contains a list of schemas in form {"<schema id>",<schema>}
 
@@ -154,7 +190,7 @@ Test example:
 
 The levenshtein distance between two character strings.
 
-![Levenshtein Distance](documentation/levenshteinDistance.JPG "Levenshtein Distance")
+![Levenshtein Distance](documentation/levenshteinDistance.jpg "Levenshtein Distance")
 
 ------------------------------------------------------------
 
@@ -242,6 +278,12 @@ Test/example flow in test/generalTest.json
 
 # Version
 
+0.18.0 Add matrix node
+
+0.17.0 Add finished wire to load injector
+
+0.16.10 data analysis add eulcidean distance functions. Add array pairs
+
 0.16.0 fix data analysis variance and stddev, add sample, add tests
 
 0.14.2 fix general test flows. Add icon for data analysis
@@ -258,40 +300,6 @@ Test/example flow in test/generalTest.json
 
 0.11.0 Transform for AVRO and snappy.  Add JSON to CSV 
 
-0.10.2 Transform validate for array source, bug fixes on transform and add improvements to array to messages. Added node for levenshtein distance.
-
-0.10.1 Real time weighted moving average, levenshtein Distance, for test allow testing of "infinity","-infinity" and "NaN" in JSON.
-
-0.10.0 Many fixes to transform.  Array and csv to various forms work.  Added test to validate.
-Improved test to allow for escape to put special characters into a string.
-
-0.9.6 Enhance transform with csv ignore lead or trailing lines.
-Add Array and CSV to Messages. Add in topic override
-
-0.9.5 Enhance transform with path and setting source and target.
-Outlier allowed to set number of deviations if median and reset or set stats
-Add outlier detection and Pearson R realtime metrics.
-
-0.8.1 Add realtime metrics to data analysis. 
-
-0.7.1
-* fix json to table html and minor code improvements.
-turn off debug mode on spawn process.
-clear down timer on close for host available
-* add Host Available
-
-0.6.0
-* add Spawn Process
-* improve experimental transform with json to table html
-
-0.5.0
-* test node add select property tested for result
-* dataAnalysis add property analysed 
-* add experimental transform
-
-0.4.0 Add test, monitor flow, data analysis
-
-0.0.1 base
 
 # Author
 

package/dataAnalysis/arrayLast.js

@@ -0,0 +1,8 @@
+if(Array.prototype.last==null)
+	Object.defineProperty(Array.prototype, "last", {
+		value() {
+			return this[this.length-1];
+		},
+		writable: true,
+		configurable: true
+	});

package/dataAnalysis/arrayPairs.js

@@ -0,0 +1,15 @@
+if(Array.prototype.pairs==null)
+	Array.prototype.pairs=function (func) {
+		const l=this.length,li=l-1;
+		if(func) {
+			for (let i=0; i<li ; i++) {
+				for (let j =i+1; j<l; j++) {
+					func(this[i], this[j],i,j,this);
+				}
+			}
+		} else {
+			const r=[];
+			this.pairs(func===null? (p1,p2,i1,i2)=>r.push([p1,p2,i1,i2]) : (p1,p2)=>r.push([p1,p2]) )
+			return r;
+		}
+	}

package/dataAnalysis/dataAnalysis.html

@@ -6,6 +6,7 @@
         defaults: {
             name: {value:"",required:false},
 		    action: {value:"avg",required:true},
+		    columns:{value:"",required:false},
 		    outputs: {value:(["realtime","realtimePredict"].includes(this.action)?3:2),required:true},
 		    outliersBase: {value:"avg",required:false},
 		    outliersStdDevs: {value:"3",required:false},
@@ -44,6 +45,11 @@
         		}
            	});
 	       	$("#node-input-action").change(function() {
+        		 if(["distances","distancesMax","distancesMin"].includes( $(this).val() )) {
+        			$(".form-row-http-in-columns").show(); 
+        		 } else {
+        			$(".form-row-http-in-columns").hide();
+        		 }
         		 if(["movingAvgSimple","movingAvgWeighted","movingAvgExponential","realtime","realtimePredict"].includes( $(this).val() )) {
         			$(".form-row-http-in-term").show(); 
         		 } else {
@@ -142,6 +148,9 @@
             <option value="corelationship">Corelationship</option>
             <option value="deltas">Deltas</option>
             <option value="deltaNormalised">Deltas Normalised</option>
+            <option value="distances">Euclidean Distances</option>
+            <option value="distancesMax">Euclidean Distances Max</option>
+            <option value="distancesMin">Euclidean Distances Min</option>
             <option value="pearsonR">Linear Correlation Coefficient</option>
             <option value="max">Maximum</option>
             <option value="avg">Mean</option>
@@ -166,6 +175,11 @@
         </select>
     </div>
 
+    <div class="form-row form-row-http-in-columns hide">
+        <label for="node-input-columns"><i class="icon-bookmark"></i> Columns</label>
+        <input type="text" id="node-input-columns" placeholder="columns">
+    </div>
+
     <div class="form-row form-row-http-in-term hide">
         <label for="node-input-term"><i class="icon-bookmark"></i> Term</label>
         <input type="number" id="node-input-term" placeholder="term" min="2" max="100" step="1">
@@ -231,4 +245,8 @@ If real-time stats then a message can send directive instruction in topic:
 			<li>3 standard deviations of the mean (or median), around 99.7%
 		</ul>	
 	</p>
+	<p>
+	Distance functions take in an array of points in an array and calculates the euclidean distance.
+	e.g. max input [ [1,1],[2,2],[3,3],[4,5]]), has output [{distance:5,points:[[1,1],[4,5]],index:[0,3]}]
+	</p>
 </script>

package/dataAnalysis/dataAnalysis.js

@@ -1,13 +1,9 @@
 const logger = new (require("node-red-contrib-logger"))("Data Analysis");
 logger.sendInfo("Copyright 2020 Jaroslav Peter Prib");
 
-Object.defineProperty(Array.prototype, "last", {
-	value() {
-		return this[this.length-1];
-	},
-	writable: true,
-	configurable: true
-});
+require("./arrayLast");
+const ed=require("./euclideanDistance.js");
+require("./forNestedEach");
 
 function crossNormalisedDeltas() {
 	const dataPoints=this.dataPoint;
@@ -120,8 +116,18 @@ function setDataPoint(value,term,node,dp) {
 	dp.weightedMovingAvg=(dp.weightedMovingAvg*2/count)/(count+1);
 	dp.exponentialWeightedMoving.forEach(c=>c.sample(value));
 }
+function getColumns(node) {
+	if(node.columns) {
+		if(node.columns instanceof Array) return node.columns
+		return eval("["+node.columns+"]");
+	}
+}
 functions={
 	avg:(d)=>functions.sum(d)/d.length,
+	arrayMax:(d)=>{  // not tested
+		let max=[],indices
+		a.forNestedEach((e,f,l)=>{const i=l[l.length-1];if(max[l]<e) {max=e,indices=l}})
+	},
 	covariance: (d)=>{
 		const means=[],covars=[],dl=d.length,dlminus1=dl-1,N=d[0].length;
 		d.forEach((e,i)=>{
@@ -141,11 +147,16 @@ functions={
 	corelationship:(d)=>{
 		const covars=functions.covariance(d);
 		const stdDev=d.map(c=>functions.stdDev(c));
-		logger.send({label:"test",d:d,covars:covars,stdDev:stdDev});
-		return 1//covars.map((a)=>a.map((c,i)=>c==null?null:c/(stdDev[i+1]*stdDev[i])));
+		return covars.map((a)=>a.map((c,i)=>c==null?null:c/(stdDev[i+1]*stdDev[i])));
 	},
 	deltas :(d)=>d.map( (c,i)=>c-(d[i-1]||0) ),
 	deltaNormalised :(d)=>d.map( (c,i)=>(c-(d[i-1]||0)) / (d[i-1]||0) ),
+	distances: (d,term,node)=>{
+		if(node.columns) return ed.distances(d,getColumns(node))
+		return ed.distances(d);
+	},
+	distancesMin: (d,term,node)=>ed.minDistances(d,getColumns(node)),
+	distancesMax: (d,term,node)=>ed.maxDistances(d,getColumns(node)),
 	max: (d)=> Math.max(...d),
 	median:(d)=>{
 		const i=Math.floor(d.length/2);
@@ -384,6 +395,7 @@ module.exports = function (RED) {
 					break;
 				}
 			} catch(ex) {
+				if(logger.active) logger.send({label:"error input",action:node.action,message:ex.message,stack:ex.stack});
 				msg.error=ex.message;
 				if(node.maxErrorDisplay) {
 					--node.maxErrorDisplay;

package/dataAnalysis/euclideanDistance.js

@@ -0,0 +1,54 @@
+require("./arrayPairs");
+function distance(point1,point2) {
+	try{
+		if(point1==null  | point2==null | point2.length==0 ) return NaN;
+		if(point1.length!==point2.length) throw Error("argument 1 array size not equal argument 2 array size") ;
+		const sum=point1.reduce((s,c,i)=>{
+			const d=c-point2[i];
+			return s+d*d;
+		},0);
+		return Math.sqrt(sum);
+	} catch(ex){
+		if(!(point1 instanceof Array)) throw Error("argument 1 not array") ;
+		if(!(point2 instanceof Array)) throw Error("argument 2 not array") ;
+		throw ex;
+	}
+}
+
+function distanceColumns(point1,point2,columns=[]) {
+	const sum=columns.reduce((s,c)=>{
+		const d=point1[c]-point2[c];
+		return s+d*d;
+	},0);
+	return Math.sqrt(sum)
+}
+function arrayPairs(points,f=distance,args) {
+	if(!(points instanceof Array)) throw Error("argument 1 not array") ;
+	const r=[];
+	points.pairs((p1,p2,i1,i2)=>{
+		r.push([ f(p1,p2,args), i1, i2])
+	});
+	return r;
+}
+function arrayPairsSet(points,f=distance,args,sf=(a,b)=>a>b,maxSetSize) {
+	let m,r=[],pi1,pi2;
+	points.pairs((p1,p2,i1,i2)=>{
+		const d=f(p1,p2,args);
+		if(m===d) {
+			r.push({points:[p1,p2],index:[i1,i2],distance:d});
+			if(maxSetSize && maxSetSize>r.length) r.shift
+		} else if(m==null | sf(m,d)==true) {
+			m=d;
+			r=[{points:[p1,p2],index:[i1,i2],distance:d}];
+		}
+	});
+	return r;
+}
+module.exports={
+		distance:distance,
+		distanceColumns:distanceColumns,
+		distances:arrayPairs,
+		distancesColumns:(a,columns)=>arrayPairs(a,distanceColumns,columns),
+		minDistances:(a,columns)=>arrayPairsSet(a,	columns?distanceColumns:distance,	columns,	(a,b)=>a>b),
+		maxDistances:(a,columns)=>arrayPairsSet(a,	columns?distanceColumns:distance,	columns,	(a,b)=>a<b)
+};

package/dataAnalysis/forNestedEach.js

@@ -0,0 +1,24 @@
+if(Array.prototype.forNestedEach==null)
+	Array.prototype.forNestedEach=function (func,filter,level=[]) {
+		const l=level.length;
+		if(filter.hasOwnProperty(l)) {
+			const e=filter[l];
+			const callFilter=(v)=>!e.includes(v);
+			this.forEach((element, index, array) => {
+				if(callFilter(index)) return;
+				if(element instanceof Array) {
+					element.forNestedEach(func,filter,level.push(index))
+				} else {
+					func(element,index,array,level)
+				}
+			} )
+			return
+		}
+		this.forEach((element, index, array) => {
+			if(element instanceof Array) {
+				element.forNestedEach(func,filter,level.push(index))
+			} else {
+				func(element,index,array,level)
+			}
+		} )
+	}