node-red-contrib-prib-functions 0.18.0 → 0.20.4

package/.github/workflows/codeql-analysis.yml

@@ -35,7 +35,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

package/.github/workflows/npmpublish.yml

@@ -11,10 +11,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 12
+          node-version: 'latest'
       - run: npm ci
       - run: npm test
 
@@ -22,10 +22,10 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: 12
+          node-version: 'latest'
           registry-url: https://registry.npmjs.org/
       - run: npm ci
       - run: npm publish

package/.vs/VSWorkspaceState.json

@@ -0,0 +1,7 @@
+{
+  "ExpandedNodes": [
+    ""
+  ],
+  "SelectedNode": "\\C:\\Users\\peter\\Source\\Repos\\node-red-contrib-prib-functions",
+  "PreviewInSolutionExplorer": false
+}

package/README.md

@@ -44,6 +44,9 @@ Single value metrics:
 Array metrics:
 * Deltas
 * Deltas Normalised
+* difference
+* difference seasonal
+* difference seasonal second order
 * Moving Average Simple (SMA)
 * Moving Average Cumulative (CMA)
 * Moving Average Weighted (WMA)
@@ -53,8 +56,9 @@ Array metrics:
 
 Array data
 * distances 
-* minimum distance(s) between points
-* maximum distance(s) between points
+* distance(s) minimum between points
+* distance(s) maximum between points
+
 
 ![Data Analysis](documentation/DataAnalysis.JPG "Data Analysis")
 
@@ -111,8 +115,12 @@ Transformations:
 *	Array to ISO8385
 *	Array to Messages
 *	Array to xlsx / xlsx object (excel uses [xlsx][7])  
-*	AVRO to JSON (uses [avsc][6]) 
+*	AVRO to JSON (uses [avsc][6])
+*   Buffer to comprossed 
 *	Confluence to JSON
+*   Compressed to Buffer
+*   Compressed to String
+*   COmpressed to JSON
 *	CSV to Array
 *	CSV to HTML
 *	CSV to Messages
@@ -199,7 +207,13 @@ The levenshtein distance between two character strings.
 Inject messages for a set period of time with varying think time.
 Primary purpose is testing and useful for load/stress testing.
 
-![Load Injector](documentation/LoadInjector.JPG "Load Injector")
+Has 3 extra data types
+1. generated id - Unique id for each message
+2. generated data - string text generated using [dummy-json][3]
+3. generated json - json generated using [dummy-json][3]
+
+
+![Load Injector](documentation/loadInjector.png "Load Injector")
 
 Test example:
 
@@ -278,27 +292,14 @@ Test/example flow in test/generalTest.json
 
 # Version
 
+0.19.0 Improve load injector, fix bug in test comparing buffers, add compression tranforms
+
 0.18.0 Add matrix node
 
 0.17.0 Add finished wire to load injector
 
-0.16.10 data analysis add eulcidean distance functions. Add array pairs
-
 0.16.0 fix data analysis variance and stddev, add sample, add tests
 
-0.14.2 fix general test flows. Add icon for data analysis
-
-0.14.1 fix capitalization issue with levenshtein Distance
-
-0.14.0 add xlsx transformer
-
-0.13.0 Confluence AVRO message converter
-
-0.12.0 added xml to json and json to xml.  Make snappy installed separately as can cause install issues 
-
-0.11.1 Fix spelling of AVRO. 
-
-0.11.0 Transform for AVRO and snappy.  Add JSON to CSV 
 
 
 # Author
@@ -318,3 +319,5 @@ Test/example flow in test/generalTest.json
 [6]: https://www.npmjs.com/package/avsc "avsc"
 
 [7]: https://www.npmjs.com/package/xlsx "xlsx"
+
+[8]: https://www.npmjs.com/package/dummy-json "dummy-json"

package/arima/index.js

@@ -0,0 +1,18 @@
+const observations=[]
+
+        
+
+if(Array.prototype.differenceSeasonalSecondOrder ==null)
+    Object.defineProperty(Array.prototype, "differenceSeasonalSecondOrder", {
+        value(lag) {
+            let i=result.length-lag-1
+            const result=new Array(size)
+            while(i--) {
+                const ilag=i-lag
+                result[i]=this[i]-this[i+1]-this[iLag]+this[iLag+1]
+            }
+            return result;
+        },
+        writable: true,
+         configurable: true
+    });

package/dataAnalysis/arrayAllRowsSwap.js

@@ -0,0 +1,15 @@
+if(Array.prototype.allRowsSwap==null)
+	  Object.defineProperty(Array.prototype, "allRowsSwap", {
+		  value(x,y) {
+			const l=this.length;
+			for(let i=0; i<l; i++) {
+				const row=this[i]
+        		const temp = row[x];
+        		row[x] = row[y];
+        		row[y] = temp;
+			}
+  			return this;
+		  },
+		  writable: true,
+		  configurable: true
+	  });

package/dataAnalysis/arrayCompareToPrecision.js

@@ -0,0 +1,34 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+	if(object.prototype.compareToPrecision==null)
+		Object.defineProperty(object.prototype, "compareToPrecision", {
+			value(toArray,precision=6) {
+				const basePrecision=Math.pow(10,-precision);
+				if(this.length!=toArray.length) throw Error("array length different")
+				this.forEach((element,index)=>{
+					const toElement=toArray[index];
+					if(typeof element !== typeof toElement) throw Error("types no same for "+index)
+					if(typeof element == "array"){
+						element.compareToPrecision(toElement,precision)
+						return
+					}
+					switch (typeof element) {
+					case "array":
+						try{
+							element.compareRounded(toElement,precision)
+						} catch(ex) {
+							throw Error(index+","+ex.message)
+						}
+						return;
+					default:
+						const diff = Math.abs(element-toElement) ;
+//						console.log({diff:diff,basePrecision:basePrecision})
+						if(Math.abs(element-toElement)/element > basePrecision)
+							throw Error(index+ " failed "+element+" <> "+toElement);
+					}
+				});
+			},
+			writable: true,
+			configurable: true
+		});
+		
+});

package/dataAnalysis/arrayDifference.js

@@ -0,0 +1,14 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+    if(object.prototype.difference==null)
+        Object.defineProperty(object.prototype, "difference", {
+            value(order=0) {
+                let i=this.length
+                if(i<2) return new object()
+                const result=new object(i-1)
+                while(--i) result[i-1]=this[i]-this[i-1];
+                return order?result.difference(--order):result;
+            },
+            writable: true,
+             configurable: true
+        });
+})

package/dataAnalysis/arrayDifferenceSeasonal.js

@@ -0,0 +1,15 @@
+
+[Array,Float32Array,Float64Array].forEach(object=>{
+    if(object.prototype.differenceSeasonal==null)
+        Object.defineProperty(object.prototype, "differenceSeasonal", {
+            value(lag=1,order=0) {
+                let i=this.length-lag
+                if(i<1) return new object()
+                const result=new object(i)
+                while(--i>=0) result[i]=this[i+lag]-this[i];
+                return order?result.difference(--order):result;
+            },
+            writable: true,
+             configurable: true
+        });
+})

package/dataAnalysis/arrayDifferenceSeasonalSecondOrder.js

@@ -0,0 +1,20 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+    if(object.prototype.differenceSeasonalSecondOrder==null)
+        Object.defineProperty(object.prototype, "differenceSeasonalSecondOrder", {
+            value(lag=1) {
+                let i=this.length-lag-1
+                if(i<1) return new object()
+                const result=new object(i)
+                let lastDiff=this[i+lag-1]-this[i-1]
+                while(--i>=0){
+                    const newDiff=this[i+lag-1]-this[i-1]
+                    result[i]=lastDiff-newDiff
+                    lastDiff=newDiff
+                }
+                return result
+            },
+            writable: true,
+             configurable: true
+        });
+})
+

package/dataAnalysis/arrayDifferenceSecondOrder.js

@@ -0,0 +1,14 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+    if(object.prototype.differenceSecondOrder==null)
+        Object.defineProperty(object.prototype, "differenceSecondOrder", {
+            value() {
+                    let i=this.length-1
+                    if(i<2) return new object()
+                    const result=new object(i-1)
+                    while(--i) result[i-1]=this[i-1]-2*this[i]+this[i+1]
+                    return result;
+                },
+                writable: true,
+                 configurable: true
+            });
+})

package/dataAnalysis/arrayForEachRange.js

@@ -0,0 +1,38 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+	if(object.prototype.forEachRange==null)
+		Object.defineProperty(object.prototype, "forEachRange", {
+			value(start=0,end=this.length-1,func,o=this) {
+				if(start<=end) {
+					if(end>=this.length) throw Error("end offset "+end+" >= array length "+this.length);
+					if(start<0) throw Error("start offset "+start+"< 0");
+					for(let i=start;i<=end;i++) func(this[i],i,this)
+				} else {
+					if(start>=this.length) throw Error("end<start and start offset "+start+" >= array length "+this.length);
+					if(end<0) throw Error("end<start and end offset "+end+"< 0")
+					for(let i=start;i>=end;i--) func(this[i],i,this)
+				}
+				return this;
+			},
+			writable: true,
+			configurable: true
+		});
+	if(object.prototype.forEachRangeNotTested==null)
+		Object.defineProperty(object.prototype, "forEachRangeNotTested", {
+			value(start=0,end=this.length-1,func,o=this) {
+				if(start<=end) {
+					if(end>=this.length) end=this.length-1;
+					if(start<0) start=0;
+					for(let i=start;i<=end;i++) func(this[i],i,this)
+				} else {
+					if(start>=this.length) start=this.length-1
+					if(end<0) end=0
+					for(let i=start;i>=end;i--) func(this[i],i,this)
+				}
+				return this;
+			},
+			writable: true,
+			configurable: true
+		});
+
+});
+

package/dataAnalysis/arrayOverlay.js

@@ -0,0 +1,13 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+    if(object.prototype.overlay==null)
+    	Object.defineProperty(object.prototype, "overlay", {
+	    	value(array) {
+             if(array[0] instanceof Array)
+                 array.forEach((element,index)=>this[index].overlay(element));
+             else
+                 array.forEach((element,index)=>this[index]=element);
+	    	},
+	    	writable: true,
+	    	configurable: true
+	    });
+});

package/dataAnalysis/arrayProduct.js

@@ -0,0 +1,11 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+	if(object.prototype.product==null)
+		Object.defineProperty(object.prototype, "product", {
+			value(i=0,end=this.length-1,product=this[i]) {
+				for(i++;i<=end;i++) product*=this[i];
+				return product;
+			},
+			writable: true,
+			configurable: true
+		});
+})

package/dataAnalysis/arrayRandom.js

@@ -0,0 +1,14 @@
+
+[Array,Float32Array,Float64Array].forEach(object=>{
+    if(object.prototype.random==null)
+        Object.defineProperty(object.prototype, "random", {
+            value(size=this.length) {
+                let i=size
+                const result=new object(i)
+                while(i--) result[i]=Math.random();
+                return result;
+            },
+            writable: true,
+            configurable: true
+        })
+})

package/dataAnalysis/arrayReduceRange.js

@@ -0,0 +1,11 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+	if(object.prototype.reduceRange==null)
+		Object.defineProperty(object.prototype, "reduceRange", {
+			value(start=0,end=this.length-1,func=(aggregate,cell,index)=>aggregate+cell,aggregate=0,o=this) {
+				for(let i=start;i<=end;i++) aggregate=func(aggregate,this[i],i,this)
+				return aggregate;
+			},
+			writable: true,
+			configurable: true
+		});
+})

package/dataAnalysis/arrayScale.js

@@ -0,0 +1,11 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+	if(object.prototype.scale==null)
+		Object.defineProperty(object.prototype, "scale", {
+			value(factor,start=0,end=this.length-1) {
+				for(let i=start;i<=end;i++) this[i]*=factor;
+				return this;
+			},
+			writable: true,
+			configurable: true
+		});
+})

package/dataAnalysis/arraySum.js

@@ -0,0 +1,11 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+	if(object.prototype.sum==null)
+		Object.defineProperty(object.prototype, "sum", {
+			value(i=0,end=this.length-1,sum=0) {
+				for(;i<=end;i++) sum+=this[i];
+				return sum;
+			},
+			writable: true,
+			configurable: true
+		});
+})

package/dataAnalysis/arraySumSquared.js

@@ -0,0 +1,11 @@
+[Array,Float32Array,Float64Array].forEach(object=>{
+	if(object.prototype.sumSquared==null)
+		Object.defineProperty(object.prototype, "sumSquared", {
+			value(i=0,end=this.length-1,sum=0) {
+				for(;i<=end;i++) sum+=this[i]**2;
+				return sum;
+			},
+			writable: true,
+			configurable: true
+		});
+})

package/dataAnalysis/arraySwap.js

@@ -0,0 +1,11 @@
+if(Array.prototype.swap==null)
+	  Object.defineProperty(Array.prototype, "swap", {
+		  value(x,y) {
+        	const temp = this[x];
+        	this[x] = this[y];
+        	this[y] = temp;
+  			return this;
+		  },
+		  writable: true,
+		  configurable: true
+	  });

package/dataAnalysis/dataAnalysis.html

@@ -7,6 +7,7 @@
             name: {value:"",required:false},
 		    action: {value:"avg",required:true},
 		    columns:{value:"",required:false},
+		    lag: {value:1,required:true},
 		    outputs: {value:(["realtime","realtimePredict"].includes(this.action)?3:2),required:true},
 		    outliersBase: {value:"avg",required:false},
 		    outliersStdDevs: {value:"3",required:false},
@@ -45,50 +46,56 @@
         		}
            	});
 	       	$("#node-input-action").change(function() {
-        		 if(["distances","distancesMax","distancesMin"].includes( $(this).val() )) {
+				if(["differenceSeasonal","differenceSeasonalSecondOrder"].includes( $(this).val() )) {
+        			$(".form-row-http-in-lag").show(); 
+        		} else {
+        			$(".form-row-http-in-lag").hide();
+        		}
+        		if(["distances","distancesMax","distancesMin"].includes( $(this).val() )) {
         			$(".form-row-http-in-columns").show(); 
-        		 } else {
+        		} else {
         			$(".form-row-http-in-columns").hide();
-        		 }
+        		}
         		 if(["movingAvgSimple","movingAvgWeighted","movingAvgExponential","realtime","realtimePredict"].includes( $(this).val() )) {
         			$(".form-row-http-in-term").show(); 
-        		 } else {
+        		} else {
         			$(".form-row-http-in-term").hide();
-        		 }
-        		 if(["movingAvgExponential"].includes( $(this).val() )) {
+        		}
+        		if(["movingAvgExponential"].includes( $(this).val() )) {
         			$("#node-input-term").attr({min:0,max:1,step:0.1});
 					if(node.term<0 || node.term>1) $("#node-input-term").val(0.5);
 	       			$("#node-input-term").change(function() {
 						if(node.term<0) $(this).val(0);
 						if(node.term>1) $(this).val(1);
 					});
-        		 } else {
+        		} else {
         			$("#node-input-term").attr({min:1,max:100,step:1});
 	       			$("#node-input-term").change(function() {
 						if(node.term<1) $(this).val(1);
 						if(node.term>100) $(this).val(100);
 					});
-        		 }        		 
-        		 if(["realtime","realtimePredict"].includes( $(this).val() )) {
+        		}        		 
+        		if(["realtime","realtimePredict"].includes( $(this).val() )) {
 	        		node.outputs=3;
            			$(".form-row-http-in-keyProperty").show();
            			$(".form-row-http-in-outliersStdDevs").show();
            			$(".form-row-http-in-outliersBase").show();
 					$("#node-input-outliersStdDevs").change();
-           		 } else {
+           		} else {
            			node.outputs=2;
            			$(".form-row-http-in-keyProperty").hide();
            			$(".form-row-http-in-outliersStdDevs").hide();
            			$(".form-row-http-in-outliersBase").hide();
-        		 }        		 
-        		 if(["pearsonR","covariance","corelationship"].includes( $(this).val() )) {
+        		}        		 
+        		if(["pearsonR","covariance","corelationship"].includes( $(this).val() )) {
            			$(".node-input-dataProperties-container-row").show();
            			$(".form-row-http-in-dataProperty").hide();
-           		 } else {
+           		} else {
            			$(".node-input-dataProperties-container-row").hide();
            			$(".form-row-http-in-dataProperty").show();
-        		 }        		 
+        		}        		 
 				$("#node-input-term").change();
+				$("#node-input-lag").change();
         	});
 	       	
   	    	node.dataProperties.forEach((r)=>addDataProperty(r));
@@ -148,6 +155,10 @@
             <option value="corelationship">Corelationship</option>
             <option value="deltas">Deltas</option>
             <option value="deltaNormalised">Deltas Normalised</option>
+			<option value="difference">Difference</option>
+			<option value="differenceSeasonal">Difference Seasonal</option>
+			<option value="differenceSeasonalSecondOrder">Difference Seasonal Second Order</option>
+			<option value="differenceSecondOrder">Difference Second Order</option>
             <option value="distances">Euclidean Distances</option>
             <option value="distancesMax">Euclidean Distances Max</option>
             <option value="distancesMin">Euclidean Distances Min</option>
@@ -162,6 +173,7 @@
             <option value="movingAvgExponential">Moving Average Exponential (EMA/EWMA)</option>
             <option value="normalize">Normalise</option>
             <option value="pearsonR">Pearson Product Moment Correlation (PPMC)</option>
+			<option value="randomise">Randomise</option>
             <option value="range">Range</option>
             <option value="realtime">RealTime Metrics</option>
             <option value="realtimePredict">RealTime Metrics + predicts</option>
@@ -179,6 +191,11 @@
         <label for="node-input-columns"><i class="icon-bookmark"></i> Columns</label>
         <input type="text" id="node-input-columns" placeholder="columns">
     </div>
+    
+	<div class="form-row form-row-http-in-lag hide">
+        <label for="node-input-lag"><i class="icon-bookmark"></i> Lag</label>
+        <input type="number" id="node-input-lag" placeholder="lag" min="1" max="10000" step="1">
+    </div>
 
     <div class="form-row form-row-http-in-term hide">
         <label for="node-input-term"><i class="icon-bookmark"></i> Term</label>

package/dataAnalysis/dataAnalysis.js

@@ -2,6 +2,11 @@ const logger = new (require("node-red-contrib-logger"))("Data Analysis");
 logger.sendInfo("Copyright 2020 Jaroslav Peter Prib");
 
 require("./arrayLast");
+require("./arrayDifference")
+require("./arrayDifferenceSeasonal")
+require("./arrayDifferenceSeasonalSecondOrder")
+require("./arrayDifferenceSecondOrder")
+require("./arrayRandom")
 const ed=require("./euclideanDistance.js");
 require("./forNestedEach");
 
@@ -157,6 +162,10 @@ functions={
 	},
 	distancesMin: (d,term,node)=>ed.minDistances(d,getColumns(node)),
 	distancesMax: (d,term,node)=>ed.maxDistances(d,getColumns(node)),
+	difference: (d)=>d.difference(),
+	differenceSeasonal: (d,term,node)=>d.differenceSeasonal(node.lag),
+	differenceSeasonalSecondOrder: (d,term,node)=>d.differenceSeasonalSecondOrder(node.lag),
+	differenceSecondOrder: (d)=>d.differenceSecondOrder(),
 	max: (d)=> Math.max(...d),
 	median:(d)=>{
 		const i=Math.floor(d.length/2);
@@ -313,7 +322,7 @@ module.exports = function (RED) {
 			}
 			switch (node.action) {
 			case "realtime":
-					node.outliersStdDevs=Number.parseInt(node.outliersStdDevs,10)||3;
+				node.outliersStdDevs=Number.parseInt(node.outliersStdDevs,10)||3;
 				if(![1,2,3].includes(node.outliersStdDevs)) throw Error("outlier std deviation "+node.outliersStdDevs+" not 1,2 or 3");
 				const outliersFunction=(node.outliersBase||"avg")=="median";
 				node.log("realtime outliersBase set avg "+outliersFunction);