@datagrok/eda 1.0.3

package/wasm/callWasmForWebWorker.js

@@ -0,0 +1,706 @@
+// Utilities for calling wasm-functions via webworker.
+
+// We use an approach that is well described here:
+// https://becominghuman.ai/passing-and-returning-webassembly-array-parameters-a0f572c65d97
+// It has been modified for usage in DATAGROK.
+
+// Constants for wasm-functions in webworkers runtime system
+const TYPE = 'type';
+const NUM_TYPE = 'num';
+const FLOAT_COLUMN_TYPE = 'floatColumn';
+const INT_COLUMN_TYPE = 'intColumn';
+const FLOAT_COLUMNS_TYPE = 'floatColumns';
+const NEW_FLOAT_COLUMNS_TYPE = 'newFloatColumns';
+const INT_COLUMNS_TYPE = 'intColumns';
+const NEW_INT_COLUMNS_TYPE = 'newIntColumns';
+const NEW_FLOAT_COLUMN_TYPE = 'newFloatColumn';
+const NEW_INT_COLUMN_TYPE = 'newIntColumn';
+const COLUMN = 'column';
+const CALL_RESULT = '_callResult';
+const NUM_OF_ROWS = 'numOfRows';
+const NUM_OF_COLUMNS = 'numOfColumns';
+const REF = 'ref';
+const VALUE = 'value';
+const TABLE_OF_COLUMNS = 'tableFromColumns';
+const OBJECTS = 'objects';
+const INT_TYPE = 'int';
+const DOUBLE_TYPE = 'double';
+const NUMBER = 'number';
+
+// Type-to-heap correspondence.
+// It is required for JS-module generated by Emscripten,
+// and it is used, when passing array data to/from wasm-functions.
+// More info can be found at the following link:
+// https://becominghuman.ai/passing-and-returning-webassembly-array-parameters-a0f572c65d97
+const heapMap = {
+                 'intColumn': "HEAP32",
+                 'floatColumn': "HEAPF32",
+                 'floatColumns': "HEAPF32",
+                 'newFloatColumns': "HEAPF32",
+                 'intColumns': "HEAP32",
+                 'newIntColumns': "HEAP32",
+                 'newFloatColumn': "HEAPF32",
+                 'newIntColumn': "HEAP32"
+                };
+
+// Type signature to typed array map.
+// It is used, when manipulating column(s).
+const typeMap = {
+                 'intColumn': Int32Array,
+                 'floatColumn': Float32Array,
+                 'floatColumns': Float32Array,
+                 'newFloatColumns': Float32Array,
+                 'intColumns': Int32Array,  
+                 'newIntColumns': Int32Array,
+                 'newFloatColumn': Float32Array,
+                 'newIntColumn': Int32Array
+                }; 
+
+// Type-to-shift map.
+// It is used, when passing array to/from wasm-functions.
+// More info can be found at the following link:
+// https://becominghuman.ai/passing-and-returning-webassembly-array-parameters-a0f572c65d97 
+const shiftMap = {'intColumn': 2, 
+                  'floatColumn': 2, 
+                  'floatColumns': 2,  
+                  'newFloatColumns': 2, 
+                  'intColumns': 2,  
+                  'newIntColumns': 2,
+                  'newFloatColumn': 2,
+                  'newIntColumn': 2
+                 };         
+
+// Get input for C++-function.
+// This function takes specification of arguments (argsSpecification) & input data (inputVals)
+// and returns input that will be further used in cpp/wasm-function.
+export function getCppInput(argsSpecification, inputVals) {
+    let cppFuncInput = [];
+    let ref;
+
+    // complete an input for cpp
+    let i = 0;
+    for(const key in argsSpecification) {
+        const arg = argsSpecification[key]; 
+        const type = arg.type;
+        
+        // skip auxiliry element
+        if(key === CALL_RESULT)             
+            continue;
+
+        // here, we consider each type of input
+        switch(type) {           
+ 
+            // numbers
+            case NUM_TYPE:                
+            case INT_TYPE:
+            case DOUBLE_TYPE:
+                arg.data = inputVals[i];
+                i++;
+                break;
+
+            // column
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:
+
+                let array;
+
+                // this is OK if type of column and target type coinside
+                //array = inputVals[i].getRawData();
+
+                let col = inputVals[i];
+
+                // here, we check types and perform an appropriate transform
+                if( ( (col.type === INT_TYPE) && (type === INT_COLUMN_TYPE) ) 
+                   || ( (col.type === DOUBLE_TYPE) && (type === FLOAT_COLUMN_TYPE) ) )
+                     array = col.getRawData();
+                else
+                     array = new typeMap[type](col.getRawData());
+
+                /*if(((col.type == 'int') && (type == INT_COLUMN_TYPE)) 
+                     || ((col.type == 'double') && (type == FLOAT_COLUMN_TYPE)))
+                       array = col.getRawData();
+                else
+                       array = new typeMap[type](col.getRawData());*/
+
+                // check types
+                arg.data = { 'array': array,
+                             'numOfRows': array.length};                             
+               
+                i++;
+                break;
+
+            // new column 
+            case NEW_INT_COLUMN_TYPE:
+            case NEW_FLOAT_COLUMN_TYPE:
+                let val = 0;                
+
+                ref = arg[NUM_OF_ROWS][REF];
+
+                if (argsSpecification[ref].type === NUM_TYPE)
+                    val = argsSpecification[ref].data;
+                else
+                    val = argsSpecification[ref].data[arg[NUM_OF_ROWS][VALUE]];                                
+
+                arg.data = {'numOfRows': val};
+
+                i++;
+                break;
+
+            // columns or column_list
+            case INT_COLUMNS_TYPE:
+            case FLOAT_COLUMNS_TYPE:                
+                let arrays = [];                
+
+                // this is OK if type of columns and target type coinside
+                //for(let col of inputVals[i].toList())
+                //  arrays.push(col.getRawData());
+
+                // here, we check types and perform an appropriate transform
+                for(let col of inputVals[i].toList())
+                  if( ( (col.type === INT_TYPE) && (type === INT_COLUMN_TYPE) ) 
+                    || ( (col.type === DOUBLE_TYPE) && (type === FLOAT_COLUMN_TYPE) ) )
+                    arrays.push(col.getRawData());
+                  else
+                    arrays.push(new typeMap[type](col.getRawData()));
+                
+                /*for(let col of inputVals[i].toList())
+                    if(((col.type == 'int') && (type == INT_COLUMN_TYPE)) 
+                      || ((col.type == 'double') && (type == FLOAT_COLUMN_TYPE)))
+                      arrays.push(col.getRawData());
+                    else
+                      arrays.push(new typeMap[type](col.getRawData()));*/
+
+                arg.data = { 'arrays': arrays,
+                    'numOfRows': arrays[0].length,
+                    'numOfColumns': arrays.length};
+
+                i++;  
+                break;
+
+            // new columns or new column_list
+            case NEW_INT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMNS_TYPE:
+                let val1 = 0;
+                let val2 = 0;
+
+                ref = arg[NUM_OF_ROWS][REF];
+
+                if (argsSpecification[ref].type === NUM_TYPE)
+                    val1 = argsSpecification[ref].data;
+                else
+                    val1 = argsSpecification[ref].data[arg[NUM_OF_ROWS][VALUE]];
+
+                ref = arg[NUM_OF_COLUMNS][REF];
+
+                //console.log('Ref:');
+                //console.log(ref);
+                //console.log(argsSpecification[ref].data);
+
+                if (argsSpecification[ref].type === NUM_TYPE)
+                    val2 = argsSpecification[ref].data;
+                else
+                    val2 = argsSpecification[ref].data[arg[NUM_OF_COLUMNS][VALUE]];                
+
+                arg.data = {'numOfRows': val1,
+                  'numOfColumns': val2};
+
+                i++;
+                break;            
+
+            default: 
+                return; // TODO: specify behaviour           
+            } // switch        
+
+        cppFuncInput.push(arg);
+    } // for key
+
+    //console.log('cppFuncInput:');
+    //console.log(cppFuncInput);
+
+    return cppFuncInput;
+} // getCppInput
+
+// Allocate memory for buffers for array data
+function allocateMemoryForBuffer(module, inputs) {
+    for(const arg of inputs) { 
+        const type = arg.type;
+
+        switch(type) { // Process each type of input
+
+            // numbers
+            case NUM_TYPE: 
+            case INT_TYPE:
+            case DOUBLE_TYPE:
+                break;
+
+            // column & new column
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:
+            case NEW_FLOAT_COLUMN_TYPE:
+            case NEW_INT_COLUMN_TYPE:
+                arg.data.buf = module._malloc(arg.data.numOfRows * typeMap[type].BYTES_PER_ELEMENT);
+                break;
+
+            // columns & new columns
+            case INT_COLUMNS_TYPE:             
+            case NEW_INT_COLUMNS_TYPE:
+            case FLOAT_COLUMNS_TYPE:             
+            case NEW_FLOAT_COLUMNS_TYPE: // allocation memory for columns that are created
+                arg.data.buf = module._malloc(arg.data.numOfRows * arg.data.numOfColumns 
+                    * typeMap[type].BYTES_PER_ELEMENT);
+                break;
+
+            // TODO: process other cases and mistakes
+            default:
+                break; 
+        }  
+    }
+    
+    //console.log('inputs after memory allocation:');
+    //console.log(inputs);
+} // allocateMemoryForBuffer
+
+// Get array of values that are put to wasm-function.
+function getArrOfWasmParams(inputs) {
+    let params = [];
+
+    // Process each type of input
+    for(const arg of inputs) {
+        switch (arg.type) {
+
+            // numbers
+            case NUM_TYPE:            
+            case INT_TYPE:
+            case DOUBLE_TYPE:
+                params.push(arg.data);
+                break;
+
+            // column & new column
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:
+            case NEW_FLOAT_COLUMN_TYPE:
+            case NEW_INT_COLUMN_TYPE:
+                params.push(arg.data.buf);                
+                params.push(arg.data.numOfRows);
+                break;
+            
+            // columns & new columns
+            case INT_COLUMNS_TYPE:
+            case NEW_INT_COLUMNS_TYPE:
+            case FLOAT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMNS_TYPE: 
+                params.push(arg.data.buf);
+                params.push(arg.data.numOfRows);
+                params.push(arg.data.numOfColumns);
+                break;
+        
+            // TODO: process other cases and mistakes
+            default:
+                break;
+        }
+    }
+
+    return params;
+} // getArrOfWasmParams
+
+// Get array of types that are put to wasm-function.
+function getArrOfWasmTypes(inputs) {
+    let types = [];
+
+    for(const arg of inputs) {
+        switch (arg.type) { // Process each type of input
+
+            // numbers
+            case NUM_TYPE:
+            case INT_TYPE:
+            case DOUBLE_TYPE:
+                types.push(NUMBER);
+                break;
+
+            // column & new column
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:
+            case NEW_FLOAT_COLUMN_TYPE:
+            case NEW_INT_COLUMN_TYPE:
+                types.push(NUMBER);                
+                types.push(NUMBER);
+                break;
+            
+            // columns & new columns
+            case INT_COLUMNS_TYPE:
+            case NEW_INT_COLUMNS_TYPE:
+            case FLOAT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMNS_TYPE:
+                types.push(NUMBER);
+                types.push(NUMBER);
+                types.push(NUMBER);
+                break;
+
+            // TODO: process other cases and mistakes
+            default:
+                break;
+        }
+    }
+
+    return types;
+} // getArrOfWasmTypes
+
+// Put array data to buffer
+function putDataToBuffer(module, inputs) {
+    let shift;
+    let heap;    
+    
+    for(const arg of inputs) {
+        const type = arg.type;
+
+        switch (type) { // Process each type of input
+
+            // numbers
+            case NUM_TYPE:
+            case INT_TYPE:
+            case DOUBLE_TYPE:                
+                break;
+                
+            // column
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:                
+                shift = shiftMap[type];
+                heap = module[heapMap[type]];  
+                heap.set(arg.data.array, arg.data.buf >> shift);
+                break;
+            
+            // columns
+            case INT_COLUMNS_TYPE:
+            case FLOAT_COLUMNS_TYPE:
+                shift = shiftMap[type];
+                heap = module[heapMap[type]];
+                let numOfBytes = typeMap[type].BYTES_PER_ELEMENT;
+                let buf = arg.data.buf;
+                let numOfColumns = arg.data.numOfColumns;
+                let numOfRows = arg.data.numOfRows;
+                let arrays = arg.data.arrays;
+
+                for(let i = 0; i < numOfColumns; i++)
+                    heap.set(arrays[i], (buf + i * numOfRows * numOfBytes) >> shift);
+                                
+                break;
+
+            // new column(s)
+            case NEW_INT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMN_TYPE:
+            case NEW_INT_COLUMN_TYPE:                
+                break;
+
+            // TODO: process other cases and mistakes
+            default:
+                break;
+        }
+    }
+} // putDataToBuffer
+
+// Get array data from buffer.
+function getDataFromBuffer(module, inputs) {
+
+    let heap;
+    let numOfRows;
+    let numOfCols;
+    let numOfBytes;    
+    let buf;
+
+    for(const arg of inputs) {
+        const type = arg.type;
+
+        switch (type) { // Process each type of input
+            
+            // number
+            case NUM_TYPE:
+            case INT_TYPE:
+            case DOUBLE_TYPE:
+                break;
+
+            // column(s)
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:
+            case FLOAT_COLUMNS_TYPE:  
+            case INT_COLUMNS_TYPE:          
+                break;
+
+            // new column
+            case NEW_FLOAT_COLUMN_TYPE:
+            case NEW_INT_COLUMN_TYPE:
+                heap = module[heapMap[type]];
+                numOfRows = arg.data.numOfRows;
+                numOfBytes = typeMap[type].BYTES_PER_ELEMENT;    
+                buf = arg.data.buf;
+                let array = new typeMap[type](numOfRows);
+
+                for(let j = 0; j < numOfRows; j++)
+                    array[j] = heap[buf / numOfBytes + j];
+
+                arg.array = array;
+
+                break;
+
+            // new columns
+            case NEW_INT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMNS_TYPE: 
+                heap = module[heapMap[type]];
+                numOfRows = arg.data.numOfRows;
+                numOfCols = arg.data.numOfColumns;
+                numOfBytes = typeMap[type].BYTES_PER_ELEMENT;    
+                buf = arg.data.buf;
+                let arrays = [];
+
+                for(let i = 0; i < numOfCols; i++) {
+                   let arr = new typeMap[type](numOfRows);
+
+                   for(let j = 0; j < numOfRows; j++)
+                      arr[j] = heap[buf / numOfBytes + j + i * numOfRows];               
+                
+                   arrays.push(arr);
+                }
+
+                arg.arrays = arrays;
+                
+                break;
+        
+            // TODO: process other cases and mistakes
+            default:
+                break;
+        }
+    }            
+} // getDataFromBuffer
+
+// Clear memory allocated for array data
+function clearMemoryForBuffer(module, inputs) {
+    for(const arg of inputs) 
+        switch(arg.type) { // process each type of input
+
+            // number
+            case NUM_TYPE:
+            case INT_TYPE:
+            case DOUBLE_TYPE:
+                break;
+
+            // each non-number case
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:                
+            case INT_COLUMNS_TYPE:
+            case NEW_INT_COLUMNS_TYPE:
+            case FLOAT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMN_TYPE:
+            case NEW_INT_COLUMN_TYPE:
+                module._free(arg.data.buf);
+                break;
+
+            // TODO: process other cases and mistakes
+            default:
+                break; 
+        }    
+} // clearMemoryForBuffer
+
+// Extract newly created data: new column(s) are created
+function extractNewlyCreatedData(funcSpecificationArgs, argsAfterWasmCall) {
+    // type-to-column_creator map
+    const typeToColumnCreatorMap = {'newFloatColumns': DG.Column.fromFloat32Array,
+                                    'newIntColumns': DG.Column.fromInt32Array,
+                                    'newFloatColumn': DG.Column.fromFloat32Array,
+                                    'newIntColumn': DG.Column.fromInt32Array};
+
+    let i = 0;
+
+    for(const key in funcSpecificationArgs) 
+    {
+        const arg = funcSpecificationArgs[key];
+
+        switch(arg.type){ // Process each type
+
+            // number
+            case NUM_TYPE:
+            case INT_TYPE:
+            case DOUBLE_TYPE:
+                break;
+
+            // column(s)
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:
+            case FLOAT_COLUMNS_TYPE:
+            case INT_COLUMNS_TYPE:
+                break;
+
+            // new column
+            case NEW_FLOAT_COLUMN_TYPE:
+            case NEW_INT_COLUMN_TYPE:
+                let name;
+
+                // specify name for column
+                if(arg.name == undefined)
+                    name = (0).toString();
+                else 
+                    names = arg.name;
+
+                arg.column = typeToColumnCreatorMap[arg.type](name,
+                    argsAfterWasmCall[i].array);
+                break;
+
+            // new columns
+            case NEW_INT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMNS_TYPE:
+                let columns = [];
+                let length = argsAfterWasmCall[i].arrays.length;
+
+                let names = [];
+
+                // specify name for column
+                if(arg.names == undefined)
+                    for(let k = 1; k <= length; k++)
+                        names.push((k).toString());
+                else names = arg.names;
+
+                for(let j = 0; j < length; j++)
+                    columns.push(typeToColumnCreatorMap[arg.type](names[j],
+                        argsAfterWasmCall[i].arrays[j]));
+
+                arg.columns = columns;
+                
+                break;
+
+            // TODO: process other cases and mistakes
+            default:
+                break;
+        }
+
+        i++;
+    }
+} // extractNewlyCreatedData
+
+// Get output data: overall output is created
+function getOutput(funcSpecification) {  
+    let output = funcSpecification.output;
+    
+    const typeToDataFieldMap = {'newFloatColumns': 'columns',
+                                'newIntColumns': 'columns',
+                                'newFloatColumn': 'column',
+                                'newIntColumn': 'column'};
+
+    switch(output.type) {
+
+        case NUM_TYPE:
+        case INT_TYPE:
+        case DOUBLE_TYPE:
+            return funcSpecification.arguments[output.source];
+            break;
+
+        case COLUMN:
+            return funcSpecification.arguments[output.source].column;
+            break;
+
+        case TABLE_OF_COLUMNS:
+            return DG.DataFrame.fromColumns(funcSpecification.arguments[output.source].columns);
+            break;
+
+        case OBJECTS:
+            let arrayToReturn = [];
+            
+            // push data of the required arguments
+            for(let name of output.source) {
+                let arg = funcSpecification.arguments[name];
+                arrayToReturn.push(arg[typeToDataFieldMap[arg.type]]);
+            }                
+
+            return arrayToReturn;
+            break;
+
+        // TODO: process other cases and mistakes
+        default:
+            break;
+    }
+
+} // getOutput
+
+// Clear newly created data fields (new column(s))
+function clearNewlyCreatedData(funcSpecificationArgs) { 
+    for(const key in funcSpecificationArgs) {
+        const arg = funcSpecificationArgs[key];
+
+        switch (arg.type) {
+            case NUM_TYPE:
+            case INT_TYPE:
+            case DOUBLE_TYPE:
+            case INT_COLUMN_TYPE:
+            case FLOAT_COLUMN_TYPE:            
+            case INT_COLUMNS_TYPE:
+            case FLOAT_COLUMNS_TYPE:
+                break;
+
+            case NEW_INT_COLUMN_TYPE:
+            case NEW_FLOAT_COLUMN_TYPE:
+                arg.column = null;                
+                break;
+
+            case NEW_INT_COLUMNS_TYPE:
+            case NEW_FLOAT_COLUMNS_TYPE:
+                arg.columns = null;
+                break;
+        
+            // TODO: process other cases and mistakes
+            default:
+                break;
+        }
+    }
+} // clearNewlyCreatedData
+
+// THE MAIN FUNCTION: a wrapper for C++-function call
+export function cppWrapper(module, args, cppFuncName, returnType) {
+    // allocate memory for arrays that are passed to C++-function
+    allocateMemoryForBuffer(module, args);
+
+    // put data (just column(s)) to allocated buffers
+    putDataToBuffer(module, args);
+
+    // create array of parameters that are passed to C++-function
+    let params = getArrOfWasmParams(args);
+
+    //console.log('params:');
+    //console.log(params);
+
+    // create array of parameters' types that are passed to C++-function
+    let types = getArrOfWasmTypes(args);
+    
+    //console.log('types:');
+    //console.log(types);
+
+    // call wasm-function
+    let result = module.ccall(cppFuncName, returnType, types, params);
+    
+    //console.log(result);
+
+    // get data from buffers (just column(s))
+    getDataFromBuffer(module, args);        
+
+    // clear memory that was previousely allocated
+    clearMemoryForBuffer(module, args);
+
+    //console.log('done');
+
+    return result;
+}
+
+// Get the required output.
+// It takes a specification of the function and the data computed
+// and extracts the required results.
+export function getResult(funcSpecification, dataFromWebWorker) {
+    funcSpecification.arguments._callResult = dataFromWebWorker.callResult;  
+
+    extractNewlyCreatedData(funcSpecification.arguments, dataFromWebWorker.args);    
+
+    let outPut = getOutput(funcSpecification);
+
+    // Below, we remove newly created column(s), which are created at the extraction-step.
+    // It is especially required, when multiple call of wasm-functions in webworker.
+    clearNewlyCreatedData(funcSpecification.arguments);
+
+    return outPut;
+}