@ejfdelgado/ejflab-common 1.0.0

package/src/MyTemplate.js

@@ -0,0 +1,272 @@
+const { CsvFormatterFilters } = require("./CsvFormatterFilters");
+const { CsvWithFilters } = require("./CsvWithFilters");
+const { SimpleObj } = require("./SimpleObj");
+
+const pInicio = "\\$\\s*\\[\\s*for\\s+([^\\s\\]]+)\\s+in\\s+([^\\s\\]]+\\s*})\\]";
+const pFin = "\\$\\s*\\[\\s*endfor\\s*\\]";
+const pNoFor = "((?!\\$\\s*\\[for).)*?";
+const pAny = "(.*?)";
+
+const pIfInicio = "\\$\\s*\\[\\s*if\\s+([^\\]]*?)\\s*\\]";
+const pIfFin = "\\$\\s*\\[\\s*endif\\s*\\]";
+const pElse = "\\$\\s*\\[\\s*else\\s*\\]";
+
+class MyTemplate extends CsvWithFilters {
+    static interpolate(text, model) {
+        const renderer = new MyTemplate();
+        return renderer.render(text, model);
+    }
+    // ${algo} -> algo
+    static getBiggerOpenClose(text, startPattern, endPattern, openIndex, closeIndex, extractDataFun, elsePattern = null, elseIndex = 4) {
+        let startOrEnd;
+        if (elsePattern == null) {
+            startOrEnd = new RegExp("(" + startPattern + ")|(" + endPattern + ")", "gsi");
+        } else {
+            startOrEnd = new RegExp("(" + startPattern + ")|(" + endPattern + ")|(" + elsePattern + ")", "gsi");
+        }
+        let m;
+        let starterFor = null;
+        let closeFor = null;
+        let elseTag = null;
+        let closeCounter = 0;
+        do {
+            m = startOrEnd.exec(text);
+            if (m) {
+                if (m[closeIndex] !== undefined) {
+                    // Is closing
+                    if (starterFor == null) {
+                        throw Error("Template error, can not close without start");
+                    }
+                    if (closeCounter == 0) {
+                        closeFor = {
+                            startIndex: m.index,
+                            finishIndex: m.index + m[0].length
+                        };
+                        break;
+                    }
+                    closeCounter--;
+                } else if (m[openIndex] !== undefined) {
+                    // Is oppening
+                    if (starterFor == null) {
+                        starterFor = extractDataFun(m);
+                        starterFor.startIndex = m.index;
+                        starterFor.finishIndex = m.index + m[0].length;
+                    } else {
+                        closeCounter++;
+                    }
+                } else if (m[elseIndex] !== undefined) {
+                    if (closeCounter == 0) {
+                        elseTag = {
+                            startIndex: m.index,
+                            finishIndex: m.index + m[0].length
+                        };
+                    }
+                }
+            }
+        } while (m);
+        if (starterFor == null) {
+            //There is no mode fors...
+            return null;
+        }
+        if (starterFor !== null && closeFor == null) {
+            throw Error(`Expected close somewhere after ${starterFor.match}`);
+        }
+        return {
+            starterFor,
+            closeFor,
+            elseTag
+        };
+    }
+    static getBiggerIf(text) {
+        const temp = MyTemplate.getBiggerOpenClose(text, pIfInicio, pIfFin, 1, 3, (m) => {
+            return {
+                formula: m[2],
+            };
+        }, pElse);
+        if (temp == null) {
+            return null;
+        }
+        const {
+            starterFor,
+            closeFor,
+            elseTag,
+        } = temp;
+        const response = {
+            formula: starterFor.formula,
+            part1: text.substring(0, starterFor.startIndex),
+            part2: text.substring(closeFor.finishIndex),
+            localTemplateFalse: "",
+        };
+
+        if (elseTag == null) {
+            response.localTemplateTrue = text.substring(starterFor.finishIndex, closeFor.startIndex);
+        } else {
+            response.localTemplateTrue = text.substring(starterFor.finishIndex, elseTag.startIndex);
+            response.localTemplateFalse = text.substring(elseTag.finishIndex, closeFor.startIndex);
+        }
+        return response;
+    }
+    static getBiggerFor(text) {
+        const temp = MyTemplate.getBiggerOpenClose(text, pInicio, pFin, 1, 4, (m) => {
+            return {
+                match: m[0],
+                varName: m[2],
+                arrayname: m[3],
+            }
+        });
+        if (temp == null) {
+            return null;
+        }
+        const {
+            starterFor,
+            closeFor
+        } = temp;
+        return {
+            varName: starterFor.varName,
+            arrayName: MyTemplate.getVariableNames(starterFor.arrayname),
+            localTemplate: text.substring(starterFor.finishIndex, closeFor.startIndex),
+            part1: text.substring(0, starterFor.startIndex),
+            part2: text.substring(closeFor.finishIndex),
+        };
+    }
+    static getVariableNames(text) {
+        return text.replace(/\$\s*\{\s*([^}]+?)\s*\}/g, "$1");
+    }
+    replaceFors(template, data) {
+        // Expand for
+        // ${for color in colores} este es mi ${color.id} ${endfor}
+        const someFor = MyTemplate.getBiggerFor(template);
+        if (someFor == null) {
+            return {
+                template,
+                times: 0,
+            };
+        }
+        const {
+            arrayName,
+            localTemplate,
+            part1,
+            part2,
+            varName
+        } = someFor;
+        let response = [];
+        const valor = SimpleObj.getValue(data, arrayName);
+        if (valor != null && typeof valor == "object") {
+            const llaves = Object.keys(valor);
+            response = llaves.map((llave) => {
+                const keyPatter = new RegExp("\\$\\s*\\{" + varName, "g");
+                //console.log(`keyPatter = ${keyPatter}`);
+                return localTemplate.replace(keyPatter, (match) => {
+                    return "${" + arrayName + "." + llave;
+                });
+            });
+        }
+        const middle = response.join("");
+        return {
+            template: part1 + middle + part2,
+            times: 1
+        };
+    }
+    computeIf(formula, data) {
+        const plainFormula = this.replaceBareValues(formula, data, "\\{", "\\}", "\\}", true);
+        //console.log(plainFormula);
+        return eval(plainFormula);// Yes it's safe here, believeme, uncomment above
+    }
+    replaceIfs(template, data) {
+        const someIf = MyTemplate.getBiggerIf(template);
+        if (someIf == null) {
+            return {
+                template,
+                times: 0,
+            };
+        }
+        //console.log(plainFormula);
+        const isTrue = this.computeIf(someIf.formula, data);
+        if (isTrue === true) {
+            return {
+                template: someIf.part1 + someIf.localTemplateTrue + someIf.part2,
+                times: 1,
+            };
+        } else {
+            return {
+                template: someIf.part1 + someIf.localTemplateFalse + someIf.part2,
+                times: 1,
+            };
+        }
+    }
+    render(template, data) {
+
+        let iteration = {};
+        do {
+            iteration = this.replaceFors(template, data);
+            template = iteration.template;
+        } while (iteration.times > 0);
+
+        // Evaluate ${if ...} ${endif}
+        do {
+            iteration = this.replaceIfs(template, data);
+            template = iteration.template;
+        } while (iteration.times > 0);
+
+        // Replace values
+        return this.replaceBareValues(template, data);
+    }
+    replaceBareValues(template, data, open = "\\{", close = "\\}", closeNoScape = "}", useStringify = false) {
+        const myPattern = new RegExp("\\$\\s*" + open + "([^" + closeNoScape + "]+)\\s*" + close, "g");
+        template = template.replace(myPattern, (match, g1) => {
+            const response = this.getColumnDescription(g1)[0];
+            const valor = SimpleObj.getValue(data, response.id);
+            if (!useStringify) {
+                const temp = this.filterValue(valor, response, undefined, response.id);
+                if (typeof temp == "object") {
+                    return JSON.stringify(temp);
+                }
+                return temp;
+            } else {
+                const temp = this.filterValue(valor, response, undefined, response.id);
+                return JSON.stringify(temp);
+            }
+        });
+        return template;
+    }
+
+    // MyTemplate.readCall("call(sound, ${val} + ${val1}, param2)", { val: 5, val1: 8 });
+    static readCall(texto, model) {
+        const renderer = new MyTemplate();
+        renderer.registerFunction("rand", CsvFormatterFilters.rand);
+        const response = {
+            action: null,
+            arguments: [],
+        }
+        const partes = /^\s*call\(\s*([^,]+)\s*(.+)\)\s*$/ig.exec(texto);
+        if (partes) {
+            response.action = partes[1];
+            const argumentos = partes[2].split(",");
+            for (let i = 0; i < argumentos.length; i++) {
+                const argumento = argumentos[i].trim();
+                if (argumento.length > 0) {
+                    try {
+                        const evaluado = renderer.render(argumento, model);
+                        try {
+                            const evaluado2 = eval(evaluado);
+                            response.arguments.push(evaluado2);
+                        } catch (err0) {
+                            response.arguments.push(evaluado);
+                        }
+                    } catch (err) {
+                        response.arguments.push(argumento);
+                    }
+                }
+            }
+        }
+        return response;
+    }
+};
+
+module.exports = {
+    MyTemplate,
+    pInicio,
+    pFin,
+    pNoFor,
+    pAny
+};

package/src/MyTemplate.test.js

@@ -0,0 +1,90 @@
+const { CsvFormatterFilters } = require("./CsvFormatterFilters");
+const { MyTemplate } = require("./MyTemplate");
+
+
+function myTest() {
+    const renderer = new MyTemplate();
+
+    const cases = [
+        {
+            txt: "my name is ${name} and I like ${colors.0.id|mapColor} ${colors.0.id|personal:1:2:3}",
+            data: { name: "Edgar", colors: [{ id: 1 }] },
+            exp: "my name is Edgar and I like red 1-1-2-3",
+        },
+        {
+            txt: "All combines: $[for color in ${colors}]$[for name in ${names}]Color ${color.id|mapColor} + ${name}, $[endfor]$[endfor]",
+            data: { names: { first: "Edgar", last: "Delgado" }, colors: [{ id: 1 }, { id: 2 }] },
+            exp: "All combines: Color red + Edgar, Color red + Delgado, Color blue + Edgar, Color blue + Delgado, ",
+        },
+        {
+            txt: "Esto es una prueba $[for parent in ${parents}] parent ${parent.name} has $[for child in ${parent.children}] ${child.name} $[endfor] $[endfor] here there is nothing",
+            data: {
+                parents: [
+                    { name: "Dog", children: [{ name: "Pepito" }, { name: "Cuco" }] },
+                    { name: "Bird", children: [{ name: "Fly" }, { name: "Shym" }] },
+                    { name: "Cat", children: [] }
+                ]
+            },
+            exp: "Esto es una prueba  parent Dog has  Pepito  Cuco   parent Bird has  Fly  Shym   parent Cat has   here there is nothing",
+        },
+        {
+            txt: "Esto es una prueba $[for parent in ${parents}] parent ${parent.name} has $[for child in ${parent.children}] ${child.name} $[endfor] $[endfor] here there is nothing",
+            data: {
+                parents: [
+                    { name: "Dog", children: undefined },
+                    { name: "Bird" },
+                    { name: "Cat", children: null }
+                ]
+            },
+            exp: "Esto es una prueba  parent Dog has   parent Bird has   parent Cat has   here there is nothing",
+        },
+        {
+            txt: 'Es $[if ${color.id} == "texto" && ${algo} === undefined && ${otro.valor} == ${color.id} ] Es uno $[endif] fin',
+            data: { color: { id: "texto" }, otro: { valor: "texto" } },
+            exp: "Es  Es uno  fin",
+        },
+        {
+            txt: "Es $[if ${color.id} == 1 && ${algo} === undefined ]Verdadero$[else]Falso$[endif] fin",
+            data: { color: { id: 2 } },
+            exp: "Es Falso fin",
+        },
+        {
+            txt: 'Es $[if ${color.id} == 1 && ${algo} === undefined ]Verdadero$[else]pero acá es $[if typeof ${color} == "object" ]Sí$[else]No$[endif]$[endif] fin',
+            data: { color: { id: 2 } },
+            exp: "Es pero acá es Sí fin",
+        },
+        {
+            txt: 'Filtrado $[for el in ${arreglo}]$[if ${el.num} >= 3]${el.num} ${el.txt}$[endif]$[endfor]',
+            data: {
+                arreglo: [
+                    { num: 1, txt: "no debe salir" },
+                    { num: 2, txt: "tampoco debe salir" },
+                    { num: 3, txt: "sí debe salir" }
+                ]
+            },
+            exp: "Filtrado 3 sí debe salir",
+        }
+    ];
+
+    renderer.registerFunction("json", CsvFormatterFilters.json);
+    renderer.registerFunction("mapColor", CsvFormatterFilters.map({ 1: "red", 2: "blue" }));
+    renderer.registerFunction("personal", (val, ...args) => {
+        return val + "-" + args.join("-");
+    });
+    renderer.registerFunction("rand", CsvFormatterFilters.rand);
+
+    for (let i = 0; i < cases.length; i++) {
+        const myCase = cases[i];
+        const actual = renderer.render(myCase.txt, myCase.data);
+        //console.log(actual);
+        if (actual !== myCase.exp) {
+            throw Error(`expected:\n${myCase.exp}\nactual:\n${actual}`);
+        } else {
+            console.log(`case ${i + 1} OK!`);
+        }
+    }
+
+    console.log(renderer.render('Todo salió bien ${person.name} ${o|rand:"azul":"amarillo":"verde"}', { person: { name: "Amigo" }, n: null }));
+}
+
+myTest();

package/src/MyTensorflow.js

@@ -0,0 +1,248 @@
+
+//import * as tf from '@tensorflow/tfjs'
+//import * as tfvis from "../../../node_modules/@tensorflow/tfjs-vis/dist/index.js";
+
+const { CsvParser } = require("./CsvParser");
+
+//3. Call front front end
+//4. Add tfvisu...
+
+class MyJSONTensorHandler {
+    constructor(someModel) {
+        this.someModel = someModel;
+        this.listener = null;
+    }
+    setListener(callback) {
+        this.listener = callback;
+    }
+    toArrayBuffer(buffer) {
+        const arrayBuffer = new ArrayBuffer(buffer.length);
+        const view = new Uint8Array(arrayBuffer);
+        for (let i = 0; i < buffer.length; ++i) {
+            view[i] = buffer[i];
+        }
+        return arrayBuffer;
+    }
+    load() {
+        const modelJSON = this.someModel;
+        const modelArtifacts = {
+            modelTopology: modelJSON.modelTopology,
+            format: modelJSON.format,
+            generatedBy: modelJSON.generatedBy,
+            convertedBy: modelJSON.convertedBy
+        };
+        if (modelJSON.weightsManifest != null) {
+            const buffer = Buffer.from(modelJSON.weightsManifest[0].binary, "base64");
+            modelArtifacts.weightSpecs = modelJSON.weightsManifest[0].weights;
+            modelArtifacts.weightData = this.toArrayBuffer(buffer);
+        }
+        if (modelJSON.trainingConfig != null) {
+            modelArtifacts.trainingConfig = modelJSON.trainingConfig;
+        }
+        if (modelJSON.userDefinedMetadata != null) {
+            modelArtifacts.userDefinedMetadata = modelJSON.userDefinedMetadata;
+        }
+        return modelArtifacts;
+    }
+    save(modelArtifacts) {
+        const binary = modelArtifacts.weightData;
+        const weightsManifest = [{
+            binary: Buffer.from(binary).toString("base64"),
+            weights: modelArtifacts.weightSpecs
+        }];
+        const modelJSON = {
+            modelTopology: modelArtifacts.modelTopology,
+            weightsManifest,
+            format: modelArtifacts.format,
+            generatedBy: modelArtifacts.generatedBy,
+            convertedBy: modelArtifacts.convertedBy
+        };
+        if (modelArtifacts.trainingConfig != null) {
+            modelJSON.trainingConfig = modelArtifacts.trainingConfig;
+        }
+        if (modelArtifacts.userDefinedMetadata != null) {
+            modelJSON.userDefinedMetadata = modelArtifacts.userDefinedMetadata;
+        }
+        if (typeof this.listener == "function") {
+            this.listener(modelJSON);
+        }
+    }
+}
+
+class MyTensorflow {
+
+    constructor() {
+        console.log("Constructor MyTensorflow");
+        this.myData = [];
+        this.myTestData = [];
+        this.metadata = null;
+        this.model = null;
+    }
+
+    setMetadata(jsonMetadata) {
+        this.metadata = jsonMetadata;
+    }
+
+    setData(textData) {
+        const parser = new CsvParser();
+        this.myData = parser.parse(textData, null, true);
+    }
+
+    setTestData(textTestData) {
+        const parser = new CsvParser();
+        this.myTestData = parser.parse(textTestData, null, true);
+    }
+
+    static clampZeroOne(val, min, max) {
+        const tam = (max - min);
+        let zeroBased = val - min;
+        if (zeroBased < 0) {
+            zeroBased = 0;
+        }
+        let zeroOne = zeroBased / tam;
+        if (zeroOne > 1) {
+            zeroOne = 1;
+        }
+        return zeroOne;
+    }
+
+    extractInputs(tf, data) {
+        const inmetadata = this.metadata.in;
+        return data.map((fila) => {
+            const resp = [];
+            for (let i = 0; i < inmetadata.length; i++) {
+                const metadata = inmetadata[i];
+                const val = fila[metadata.column];
+                const clamped = MyTensorflow.clampZeroOne(val, metadata.min, metadata.max);
+                resp.push(clamped);
+            }
+            return resp;
+        });
+    }
+
+    extractOutput(tf, data) {
+        const outmetadata = this.metadata.out;
+        const outputs = data.map((fila) => {
+            const val = fila[outmetadata.column];
+            const zeroOne = MyTensorflow.clampZeroOne(val, outmetadata.min, outmetadata.max);
+            const extended = zeroOne * (outmetadata.ngroups);
+            let response = Math.floor(extended);
+            if (response == outmetadata.ngroups) {
+                response = outmetadata.ngroups - 1;
+            }
+            return response;
+        });
+        return outputs;
+    }
+
+    prepareData(tf, data) {
+        return tf.tidy(() => {
+            const outmetadata = this.metadata.out;
+            tf.util.shuffle(data);
+            const inputs = this.extractInputs(tf, data);
+            const output = this.extractOutput(tf, data);
+            const inputTensor = tf.tensor2d(inputs, [inputs.length, inputs[0].length]);
+            const outputTensor = tf.oneHot(tf.tensor1d(output, 'int32'), outmetadata.ngroups);
+            //inputTensor.print(true);
+            //outputTensor.print(true);
+            return {
+                inputs: inputTensor,
+                outputs: outputTensor,
+            }
+        });
+    }
+
+    async validate(tf) {
+        if (this.model == null) {
+            throw Error("No hay un modelo creado");
+        }
+        const {
+            inputs,
+            outputs,
+        } = this.prepareData(tf, this.myTestData);
+        const results = await this.model.evaluate(inputs, outputs, { batchSize: 2 });
+        console.log('Accuracy is:')
+        results[1].print();
+    }
+
+    async predict(tf, valores) {
+        if (this.model == null) {
+            throw Error("No hay un modelo creado");
+        }
+        const predProb = this.model.predict(tf.tensor2d([valores])).dataSync();
+        console.log("predProb:" + JSON.stringify(predProb));
+    }
+
+    async fit(tf) {
+        if (this.model == null) {
+            throw Error("No hay un modelo creado");
+        }
+        const {
+            inputs,
+            outputs,
+        } = this.prepareData(tf, this.myData);
+
+        const X = inputs;
+        const y = outputs;
+        const data = this.metadata;
+        await this.model.fit(X, y, data.fit);
+    }
+
+    compile(tf) {
+        const data = this.metadata;
+        const myCompile = Object.assign({}, data.compile, { optimizer: tf.train.adam(0.1) });
+        this.model.compile(myCompile);
+    }
+
+    async getJsonModel(tf) {
+        if (this.model == null) {
+            throw Error("No hay un modelo creado");
+        }
+        return new Promise((resolve, reject) => {
+            const saver = new MyJSONTensorHandler();
+            saver.setListener(async (modelJson) => {
+                resolve(modelJson);
+            });
+            this.model.save(saver);
+        });
+    }
+
+    async setJsonModel(tf, modelJson) {
+        const handler = new MyJSONTensorHandler(modelJson);
+        this.model = await tf.loadLayersModel(handler);
+        this.compile(tf);
+    }
+
+    async createJsonModel(tf) {
+        const data = this.metadata;
+        this.model = tf.sequential();
+
+        for (let i = 0; i < data.layers.length; i++) {
+            const actual = data.layers[i];
+            const myLayer = {
+                name: `hidden-layer-${i + 1}`,
+                units: actual.units,
+                activation: actual.activation
+            }
+            if (i == 0) {
+                myLayer.inputShape = [data.in.length];
+            }
+            this.model.add(
+                tf.layers.dense(myLayer)
+            );
+        }
+
+        this.model.add(
+            tf.layers.dense({
+                units: data.out.ngroups,
+                activation: "softmax"
+            })
+        );
+
+        this.compile(tf);
+    }
+}
+
+module.exports = {
+    MyTensorflow
+};

package/src/MyTensorflow.test.js

@@ -0,0 +1,19 @@
+const { MyTensorflow } = require("./MyTensorflow");
+const tf = require('@tensorflow/tfjs');
+const fs = require('fs');
+
+const test = () => {
+
+    const csvText = fs.readFileSync("./data/tensordata.csv", { encoding: "utf8" });
+    const csvTestText = fs.readFileSync("./data/tensordata.test.csv", { encoding: "utf8" });
+    const jsonMeta = JSON.parse(fs.readFileSync("./data/tensordata.json", { encoding: "utf8" }));
+
+    const tensorflow = new MyTensorflow();
+    tensorflow.setMetadata(jsonMeta);
+    tensorflow.setData(csvText);
+    tensorflow.setTestData(csvTestText);
+    await tensorflow.run(tf);
+    await tensorflow.validate(tf);
+}
+
+test();

package/src/MyTensorflowBack.mjs

@@ -0,0 +1,33 @@
+import * as tf from '@tensorflow/tfjs-node'
+import { MyTensorflow } from "./MyTensorflow.js";
+import fs from 'fs'
+
+export default class MyTensorflowBack extends MyTensorflow {
+    static async test() {
+        const csvText = fs.readFileSync("./data/tensordata.csv", { encoding: "utf8" });
+        const csvTestText = fs.readFileSync("./data/tensordata.test.csv", { encoding: "utf8" });
+        const jsonMeta = JSON.parse(fs.readFileSync("./data/tensordata.json", { encoding: "utf8" }));
+        const jsonModel = JSON.parse(fs.readFileSync("./data/tensormodel.json", { encoding: "utf8" }));
+
+        let tensorflow = new MyTensorflowBack();
+        tensorflow.setMetadata(jsonMeta);
+        tensorflow.setData(csvText);
+        tensorflow.createJsonModel(tf);
+        await tensorflow.fit(tf);
+
+        //const jsonModel = await tensorflow.getJsonModel(tf);
+        //console.log(JSON.stringify(jsonModel, null, 4));
+        tensorflow = new MyTensorflowBack();
+        tensorflow.setMetadata(jsonMeta);
+        tensorflow.setData(csvText);
+        tensorflow.setTestData(csvTestText);
+        await tensorflow.setJsonModel(tf, jsonModel);
+
+        await tensorflow.fit(tf);
+
+        await tensorflow.validate(tf);
+        await tensorflow.predict(tf, [0, 0]);
+    }
+}
+
+MyTensorflowBack.test();