node-red-contrib-prib-functions 0.18.0 → 0.19.2

package/README.md

@@ -111,8 +111,12 @@ Transformations:
 *	Array to ISO8385
 *	Array to Messages
 *	Array to xlsx / xlsx object (excel uses [xlsx][7])  
-*	AVRO to JSON (uses [avsc][6]) 
+*	AVRO to JSON (uses [avsc][6])
+*   Buffer to comprossed 
 *	Confluence to JSON
+*   Compressed to Buffer
+*   Compressed to String
+*   COmpressed to JSON
 *	CSV to Array
 *	CSV to HTML
 *	CSV to Messages
@@ -199,7 +203,13 @@ The levenshtein distance between two character strings.
 Inject messages for a set period of time with varying think time.
 Primary purpose is testing and useful for load/stress testing.
 
-![Load Injector](documentation/LoadInjector.JPG "Load Injector")
+Has 3 extra data types
+1. generated id - Unique id for each message
+2. generated data - string text generated using [dummy-json][3]
+3. generated json - json generated using [dummy-json][3]
+
+
+![Load Injector](documentation/loadInjector.png "Load Injector")
 
 Test example:
 
@@ -278,27 +288,14 @@ Test/example flow in test/generalTest.json
 
 # Version
 
+0.19.0 Improve load injector, fix bug in test comparing buffers, add compression tranforms
+
 0.18.0 Add matrix node
 
 0.17.0 Add finished wire to load injector
 
-0.16.10 data analysis add eulcidean distance functions. Add array pairs
-
 0.16.0 fix data analysis variance and stddev, add sample, add tests
 
-0.14.2 fix general test flows. Add icon for data analysis
-
-0.14.1 fix capitalization issue with levenshtein Distance
-
-0.14.0 add xlsx transformer
-
-0.13.0 Confluence AVRO message converter
-
-0.12.0 added xml to json and json to xml.  Make snappy installed separately as can cause install issues 
-
-0.11.1 Fix spelling of AVRO. 
-
-0.11.0 Transform for AVRO and snappy.  Add JSON to CSV 
 
 
 # Author
@@ -318,3 +315,5 @@ Test/example flow in test/generalTest.json
 [6]: https://www.npmjs.com/package/avsc "avsc"
 
 [7]: https://www.npmjs.com/package/xlsx "xlsx"
+
+[8]: https://www.npmjs.com/package/dummy-json "dummy-json"

package/dataAnalysis/pca.js

@@ -0,0 +1,399 @@
+function PCA() {
+}
+PCA.prototype.computeDeviationMatrix=function(matrix) {
+    return this.subtract(matrix, this.scale(this.multiply(this.unitSquareMatrix(matrix.length), matrix), 1 / matrix.length));
+}
+PCA.prototype.computeDeviationScores=function(deviation) {
+    return this.multiply(this.transpose(deviation), deviation);
+}
+PCA.prototype.computeVarianceCovariance=function(devSumOfSquares, sample) {
+    return this.scale(devSumOfSquares, 1 / (devSumOfSquares.length - sample?1:0));
+}
+PCA.prototype.computeSVD=function((atrix){
+    const result = svd(matrix);
+    const eigenvectors = result.U;
+    const eigenvalues = result.S;
+    return eigenvalues.map((value,i)=>{
+        return {
+            eigenvalue:value,
+            vector:eigenvectors.map((vector,j)=>-vector[i]) //HACK prevent completely negative vectors
+        }
+    });
+}
+//     Get reduced dataset after removing some dimensions
+PCA.prototype.computeAdjustedData=function(data, ...vectorObjs){
+    const vectors = vectorObjs.map((v)=>v.vector);
+    const matrixMinusMean = this.computeDeviationMatrix(data);
+    const adjustedData = this.multiply(vectors, this.transpose(matrixMinusMean));
+    const unit = this.unitSquareMatrix(data.length);
+    const avgData = this.scale(multiply(unit, data), -1 / data.length); //NOTE get the averages to add back
+    return {
+        adjustedData: adjustedData,
+        formattedAdjustedData:formatData(adjustedData, 2),
+        avgData: avgData,
+        selectedVectors: vectors
+    };
+}
+// Get original data set from reduced data set (decompress)
+PCA.prototype.computeOriginalData=function(adjustedData, vectors, avgData) {
+    const originalWithoutMean = this.transpose(multiply(transpose(vectors), adjustedData));
+    const originalWithMean = this.subtract(originalWithoutMean, avgData);
+    return {
+        originalData: originalWithMean,
+        formattedOriginalData: this.formatData(originalWithMean, 2)
+    }
+PCA.prototype.computePercentageExplained=function(vectors, ...selected) {
+    const total = vectors.map((v)=>v.eigenvalue).reduce((a,b)=>a+b);
+    const explained = selected.map((v)=>v.eigenvalue).reduce((a,b)=>a+b);
+    return (explained / total);
+}
+PCA.prototype.getEigenVectors=function(data) {
+   return this.computeSVD(this.computeVarianceCovariance(this.computeDeviationScores(this.computeDeviationMatrix(data)), false));
+}
+PCA.prototype.analyseTopResult=function(data){
+    const eigenVectors = this.getEigenVectors(data);
+    const sorted = this.eigenVectors.sort((a, b)=>b.eigenvalue-a.eigenvalue);
+    const selected = sorted[0].vector;
+    return this.computeAdjustedData(data, selected);
+}
+PCA.prototype.formatData=function(data, precision) {
+    const factor= Math.pow(10, precision || 2);
+    return data.map((d,i)=>d.map((n)=>Math.round(n * factor) / factor))
+}
+PCA.prototype.testMatrix=function(a){
+    if(!a[0] || !a.length) throw Error("Not matrix")
+}
+PCA.prototype.multiply=function(a, b){
+    this.testMatrix(a);
+    this.testMatrix(b);
+    if(b.length !== a[0].length) throw new Error('Columns in A should be the same as the number of rows in B');
+    const product = [];
+    for(let i = 0; i < a.length; i++) {
+        product[i] = []; //initialize a new row
+        for (var j = 0; j < b[0].length; j++) {
+            for (var k = 0; k < a[0].length; k++) {
+                (product[i])[j] = !!(product[i])[j] ? (product[i])[j] + (a[i])[k] * (b[k])[j] : (a[i])[k] * (b[k])[j];
+            }
+        }
+    }
+    return product;
+}
+PCA.prototype.subtract=function(a,b) {
+    if(!(a.length === b.length && a[0].length === b[0].length)) throw Error('Both A and B should have the same dimensions');
+    const result=[];
+    for(let i=0;i< a.length;i++){
+        result[i]=[];
+        for (let j = 0; j < b[0].length; j++) {
+            (result[i])[j] = (a[i])[j] - (b[i])[j];
+       }
+    }
+    return result;
+}
+PCA.prototype.scale=function(matrix, factor) {
+    const result = [];
+    for(let i = 0; i < matrix.length; i++) {
+        result[i] = [];
+        for(let j = 0; j < matrix[0].length; j++) {
+            (result[i])[j] = (matrix[i])[j] * factor;
+        }
+    }
+    return result;
+}
+PCA.prototype.unitSquareMatrix=function(rows) {
+    const result = [];
+    for(let i = 0; i < rows; i++) {
+        result[i] = [];
+        for (var j = 0; j < rows; j++) {
+            (result[i])[j] = 1;
+        }
+    }
+    return result;
+}
+PCA.prototype.transpose=function(matrix) {
+    const operated = this.clone(matrix);
+    return operated[0].map((m,c)=>matrix.map((r)=>r[c]));
+}
+PCA.prototype.clone=function(m) {
+    return m.map(r=>r.map(c=>c));
+}
+PCA.prototype.svd=function(A) {
+        let temp;
+        const prec = Math.pow(2, -52) // double presision
+        const tolerance = 1.e-64 // precision;
+        const itmax = 50;
+        let c = 0;
+        let i = 0;
+        let j = 0;
+        let k = 0;
+        let l = 0;
+        const u = this.clone(A);
+        const m = u.length;
+        const n = u[0].length;
+        if (m < n) throw "Need more rows than columns"
+
+        const e = new Array(n); //vector1
+        const q = new Array(n); //vector2
+        for (i = 0; i < n; i++) e[i] = q[i] = 0.0;
+        var v = rep([n, n], 0);
+
+        function pythag(a, b) {
+            a = Math.abs(a)
+            b = Math.abs(b)
+            if(a > b) return a * Math.sqrt(1.0 + (b * b / a / a))
+            if(b == 0.0) return a
+            return b * Math.sqrt(1.0 + (a * a / b / b))
+        }
+        function rep(s, v, k=0) {
+            const n=s[k],
+                ret=Array(n);
+            let i;
+            if(k === s.length - 1) {
+                for(i=n-2; i >= 0; i-=2) {
+                    ret[i+1] = v;
+                    ret[i] = v;
+                }
+                if(i === -1) ret[0] = v;
+                return ret;
+            }
+            const kPlusOne=k++;
+            for (i= n-1; i>=0; i--) {
+                ret[i] = rep(s, v, kPlusOne);
+            }
+            return ret;
+        }
+        let f = 0.0;
+        let g = 0.0;
+        let h = 0.0;
+        let x = 0.0;
+        let y = 0.0;
+        let z = 0.0;
+        let s = 0.0;
+
+        for(i=0; i<n; i++) {
+            e[i] = g; //vector
+            s = 0.0; //sum
+            l = i+1; //stays i+1
+            for(j=i; j<m; j++)
+                s += (u[j][i] * u[j][i]);
+            if(s <= tolerance)
+                g = 0.0;
+            else {
+                f = u[i][i];
+                g = Math.sqrt(s);
+                if(f >= 0.0) g = -g;
+                h = f * g - s
+                u[i][i] = f - g;
+                for(j=l; j<n; j++) {
+                    s = 0.0
+                    for(k=i; k < m; k++)
+                        s += u[k][i] * u[k][j]
+                    f = s / h
+                    for(k=i; k<m; k++)
+                        u[k][j] += f * u[k][i]
+                }
+            }
+            q[i] = g
+            s = 0.0
+            for(j=l; j<n; j++)
+                s = s + u[i][j] * u[i][j]
+            if (s <= tolerance)
+                g = 0.0
+            else {
+                f = u[i][i + 1]
+                g = Math.sqrt(s)
+                if (f >= 0.0) g = -g
+                h = f * g - s
+                u[i][i + 1] = f - g;
+                for(j=l; j < n; j++) e[j] = u[i][j] / h
+                for(j=l; j < m; j++) {
+                    s = 0.0
+                    for(k=l; k<n; k++)
+                        s += (u[j][k] * u[i][k])
+                    for(k=l; k<n; k++)
+                        u[j][k] += s * e[k]
+                }
+            }
+            y = Math.abs(q[i]) + Math.abs(e[i])
+            if(y > x)
+                x = y
+        }
+
+        // accumulation of right hand transformations
+        for(i = n-1; i!=-1; i--) {
+            if (g != 0.0) {
+                h = g * u[i][i + 1]
+                for (j = l; j < n; j++)
+                    v[j][i] = u[i][j] / h //u is array, v is square of columns
+                for (j = l; j < n; j++) {
+                    s = 0.0
+                    for (k = l; k < n; k++)
+                        s += u[i][k] * v[k][j]
+                    for (k = l; k < n; k++)
+                        v[k][j] += (s * v[k][i])
+                }
+            }
+            for (j = l; j < n; j++) {
+                v[i][j] = 0;
+                v[j][i] = 0;
+            }
+            v[i][i] = 1;
+            g = e[i]
+            l = i
+        }
+
+        // accumulation of left hand transformations
+        for(i=n-1; i != -1; i += -1) {
+            l = i + 1
+            g = q[i]
+            for (j = l; j < n; j++)
+                u[i][j] = 0;
+            if (g != 0.0) {
+                h = u[i][i] * g
+                for (j = l; j < n; j++) {
+                    s = 0.0
+                    for (k = l; k < m; k++) s += u[k][i] * u[k][j];
+                    f = s / h
+                    for (k = i; k < m; k++) u[k][j] += f * u[k][i];
+                }
+                for (j = i; j < m; j++) u[j][i] = u[j][i] / g;
+            } else
+                for (j = i; j < m; j++) u[j][i] = 0;
+            u[i][i] += 1;
+        }
+
+        // diagonalization of the bidiagonal form
+        prec = prec * x
+        for(k=n-1; k != -1; k--) {
+            const kPlusOne=k+1;
+            const kMinusOne=k-1;
+            for(let iteration = 0; iteration < itmax; iteration++) { // test f splitting
+                let test_convergence = false
+                for(l=k; l != -1; l--) {
+                    if(Math.abs(e[l]) <= prec) {
+                        test_convergence = true
+                        break
+                    }
+                    if (Math.abs(q[l - 1]) <= prec)
+                        break
+                }
+                if (!test_convergence) { // cancellation of e[l] if l>0
+                    c = 0.0
+                    s = 1.0
+                    const l1 = l - 1
+                    for(i=l; i<kPlusOne; i++) {
+                        f = s * e[i]
+                        e[i] = c * e[i]
+                        if (Math.abs(f) <= prec)
+                            break
+                        g = q[i]
+                        h = pythag(f, g)
+                        q[i] = h
+                        c = g / h
+                        s = -f / h
+                        for(j = 0; j<m; j++) {
+                            y = u[j][l1]
+                            z = u[j][i]
+                            u[j][l1] = y * c + (z * s)
+                            u[j][i] = -y * s + (z * c)
+                        }
+                    }
+                }
+                // test f convergence
+                z = q[k]
+                if(l == k) { //convergence
+                    if(z < 0.0) { //q[k] is made non-negative
+                        q[k] = -z
+                        for (j=0; j<n; j++)
+                            v[j][k] = -v[j][k]
+                    }
+                    break //break out of iteration loop and move on to next k value
+                }
+                if (iteration >= itmax - 1)
+                    throw 'Error: no convergence.'
+                // shift from bottom 2x2 minor
+                x = q[l]
+                y = q[kMinusOne]
+                g = e[kMinusOne]
+                h = e[k]
+                f = ((y - z) * (y + z) + (g - h) * (g + h)) / (2.0 * h * y)
+                g = pythag(f, 1.0)
+                if(f < 0.0)
+                    f = ((x - z) * (x + z) + h * (y / (f - g) - h)) / x
+                else
+                    f = ((x - z) * (x + z) + h * (y / (f + g) - h)) / x
+                // next QR transformation
+                c = 1.0
+                s = 1.0
+                for(i=l+1; i<kPlusOne; i++) {
+                    g = e[i]
+                    y = q[i]
+                    h = s * g
+                    g = c * g
+                    z = pythag(f, h)
+                    e[i - 1] = z
+                    c = f / z
+                    s = h / z
+                    f = x * c + g * s
+                    g = -x * s + g * c
+                    h = y * s
+                    y = y * c
+                    for (j = 0; j < n; j++) {
+                        x = v[j][i - 1]
+                        z = v[j][i]
+                        v[j][i - 1] = x * c + z * s
+                        v[j][i] = -x * s + z * c
+                    }
+                    z = pythag(f, h)
+                    q[i - 1] = z
+                    c = f / z
+                    s = h / z
+                    f = c * g + s * y
+                    x = -s * g + c * y
+                    for (j = 0; j < m; j++) {
+                        y = u[j][i - 1]
+                        z = u[j][i]
+                        u[j][i - 1] = y * c + z * s
+                        u[j][i] = -y * s + z * c
+                    }
+                }
+                e[l] = 0.0
+                e[k] = f
+                q[k] = x
+            }
+        }
+        const ql=q.length;
+        for(i=0; i<ql; i++)
+            if (q[i] < prec) q[i] = 0
+
+        //sort eigenvalues	
+        for(i=0; i<n; i++) {
+            for(j=i-1; j>=0; j--) {
+                if(q[j] < q[i]) {
+                    c = q[j]
+                    q[j] = q[i]
+                    q[i] = c
+                    const ul=u.length;
+                    for(k = 0; k< ul; k++) {
+                        temp = u[k][i];
+                        u[k][i] = u[k][j];
+                        u[k][j] = temp;
+                    }
+                    const vl=v.length;
+                    for(k=0; k<vl; k++) {
+                        temp = v[k][i];
+                        v[k][i] = v[k][j];
+                        v[k][j] = temp;
+                    }
+                    i = j
+                }
+            }
+        }
+        return {
+            U: u,
+            S: q,
+            V: v
+        }
+    }
+}
+
+module.exports = PCA;

package/matrix/matrix.js

@@ -30,7 +30,12 @@ Matrix.prototype.addCell=function(row,column,value){
 	this.vector[this.getIndex(row,column)]+=value;
 	return this;
 }
-Matrix.prototype.addRow=function(vector){
+Matrix.prototype.addRow=function(row,vectorIn){
+	if(vectorIn & row>=0){
+		this.forRowCells(row,(value,column,offset,vector)=>vector[offset]+=vectorIn[column]);
+		return this;
+	}
+	const vector=vectorIn?vectorIn:row;
 	if(this.size==this.sizeMax){
 		this.vector.copyWithin(0,this.columns,this.sizeMax);
 		this.rows--;
@@ -172,7 +177,7 @@ Matrix.prototype.fillArray=function(a){
 Matrix.prototype.findColumnRow=function(column,call,startRow=0,endRow=this.rows-1){
 	let offset=startRow*this.columns+column;
 	for(let row=startRow;row<=this.rows;row++){
-		if(call.apply(this,[this.vector[offset],row,offset,this.vector])) {
+		if(call.apply(this,[this.vector[offset],row,column,offset,this.vector])) {
 			return row;
 		}
 		offset+=this.columns;
@@ -205,6 +210,17 @@ Matrix.prototype.forEachCell=function(call){
 	}
 	return this;
 }
+Matrix.prototype.forEachCellLowerTriangle=function(call){
+	this.testIsSquare();
+	for(let row=0;row<this.rows;row++) {
+		let offset=row*this.columns;
+		for(let column=0;column<=row;column++) {
+			call.apply(this,[this.vector[offset],row,column,this.vector,offset,this]);
+			offset++
+		}
+	}
+	return this;
+}
 Matrix.prototype.forEachCellPairSet=function(matrix,call){
 	if(matrix instanceof Matrix){
 		if(this.rows!=matrix.rows) throw Error("number of rows different");
@@ -223,6 +239,17 @@ Matrix.prototype.forEachCellPairSet=function(matrix,call){
 	}
 	return this;
 }
+Matrix.prototype.forEachCellUpperTriangle=function(call){
+	this.testIsSquare();
+	for(let row=0;row<this.rows;row++) {
+		let offset=row*this.columns;
+		for(let column=row;column<=this.columns;column++) {
+			call.apply(this,[this.vector[offset],row,column,this.vector,offset,this]);
+			offset++
+		}
+	}
+	return this;
+}
 Matrix.prototype.forEachRow=function(call){
 	for(let row=0;row<this.rows;row++) {
 		const rowVector=this.getRow(row);
@@ -386,6 +413,11 @@ Matrix.prototype.getRow=function(row){
 	const start=row*this.columns;
 	return this.vector.subarray(start,start+this.columns);
 }
+Matrix.prototype.getVandermonde=function(vectorIn,columns){
+    const matrix = new Matrix(vectorIn.length,columns);
+	matrix.forEachCell((cell,row,column,vector,offset,object)=>vector[offset]=vectorIn[row]**column)
+	return matrix;
+}
 Matrix.prototype.getZeroed=function(row, column){
 	const offset=row*this.columns+column;
 	const value=this.vector[offset];
@@ -439,6 +471,16 @@ Matrix.prototype.multiplyRow=function(row,factor){
 	this.forRowCells(row,(cell,column,offset,vector)=>vector[offset]*=factor)
 	return this;
 }
+Matrix.prototype.norm=function(){
+
+	return Math.sqrt(this.reduce((aggregate,cell)=>aggregate+cell*cell))
+}
+Matrix.prototype.reduce=function(call,aggregate=0){
+	this.forEachCell((cell,row,column,vector,offset,object)=>
+		aggregate=call.apply(this,[aggregate,cell,row,column,vector,offset,object])
+	);
+	return aggregate;
+}
 Matrix.prototype.reducedRowEchelonForm=function(){
 	const lastColumn=this.columns-1;
     for(let pivotColumn=0,pivotRow=0;pivotRow<this.rows;pivotRow++){
@@ -526,6 +568,10 @@ Matrix.prototype.setRow=function(vector,row){
 	this.vector.set(vector, row*this.columns);
 	return this;
 }
+Matrix.prototype.setRunningSum=function(){
+	this.forEachCellLowerTriangle((cell,row,column,vector,offset,object)=>vector[offset]=1);
+	return this;
+}
 Matrix.prototype.substract=function(matrix){
 	this.forEachCellPairSet(matrix,(cellA,cellB)=>cellA-cellB)
 	return this;