backend-plus 2.0.0-rc.9 → 2.0.2

package/lib/backend-plus.js

@@ -97,7 +97,13 @@ class AppBackend{
         /** @type {{path:string}[]} */
         this.appStack=[];
         /** @type {{message:string; fun:()=>void}[]} */
-        this.shootDownCallbackList=[];
+        this.shutdownCallbackList=[];
+        this.sessionStores={
+            file: SessionFileStore,
+            memory: memorystore,
+            memoryDevel: BindMemoryPerodicallySaved(this),
+            "memory-saved": BindMemoryPerodicallySaved(this),
+        }
         if(!this.rootPath){
             console.log('ATENCIÓN hay que poner be.rootPath antes de llamar a super()');
             this.rootPath=Path.resolve(__dirname,'..');
@@ -119,11 +125,14 @@ class AppBackend{
         this.tableStructures = {};
         this.configStaticConfig();
         /** @type {null|()=>Promise<void>} */
-        this.shootDownBackend = null;
+        this.shutdownBackend = null;
         /** @type {bp.Server} */
         // @ts-ignore
         this.server = null;
     }
+    esJavascript(type){
+        return ['js','mjs'].includes(type)
+    }
     clearCaches(){
         this.caches={
             procedures:{}
@@ -358,6 +367,8 @@ AppBackend.prototype.i18n.messages.es={
     }
 };
 
+function BindMemoryPerodicallySaved(be){
+    return (
 /**
  * @param {Express.Session} session
  */
@@ -406,7 +417,7 @@ function MemoryPerodicallySaved(session){
                 console.log('dumping sessions every',dumpEverySeconds,'seconds');
                 var errorsToShow=4;
                 // TODO: hangs the server in devel mode
-                setInterval(function(){
+                var interval = setInterval(function(){
                     try{
                         fs.writeFile(fileName,json4all.stringify(store.store.dump()));
                     }catch(err){
@@ -417,18 +428,15 @@ function MemoryPerodicallySaved(session){
                         }
                     }
                 },dumpEverySeconds*1000);
+                be.shutdownCallbackListAdd({
+                    message:'session saver',
+                    fun:()=>clearInterval(interval)
+                })
             });
         }
     }
     return MemoryDevelConstructor;
-}
-
-var sessionStores={
-    file: SessionFileStore,
-    memory: memorystore,
-    memoryDevel: MemoryPerodicallySaved,
-    "memory-saved": MemoryPerodicallySaved,
-}
+})}
 
 /**
  * @param {string} text 
@@ -456,7 +464,13 @@ AppBackend.prototype.setStaticConfig = function setStaticConfig(defConfigYamlStr
 }
 
 AppBackend.prototype.configList = function configList(){
-    var list=[this.staticConfig];
+    var list=[
+        {
+            package:{
+                version: packagejson.version
+            }
+        },
+        this.staticConfig];
     if(!this.staticConfig["client-setup"]?.title && fs.existsSync(this.rootPath+'/def-config.yaml')){
         console.log('DEPRECATED!!!!!!')
         console.error('ERROR el def-config hay que ponerlo dentro de staticConfig');
@@ -504,6 +518,11 @@ AppBackend.prototype.canChangePass = async function canChangePass(reqOrContext,
     return be.isAdmin(reqOrContext);
 }
 
+AppBackend.prototype.shutdownCallbackListAdd = function(messageFun){
+    this.shutdownCallbackList.push(messageFun);
+}
+
+
 AppBackend.prototype._Browsers = {
     Edge:      {short:'Ed'  , minVer:14  , polly:true},
     Konqueror: {short:'Kq'  , minVer:null, polly:true},
@@ -520,7 +539,7 @@ var imgExts =  ['jpg', 'png', 'jpeg', 'ico', 'gif', 'svg']
 
 AppBackend.prototype.exts = {
     img: imgExts,
-    normal: ['', 'js', 'map', 'html', 'css', ...imgExts, 'appcache', 'manifest', 'json', 'webmanifest', 'zip', 'pdf', ...fontExts, 'xlsx', 'csv']
+    normal: ['', 'js', 'map', 'html', 'css', ...imgExts, 'appcache', 'manifest', 'json', 'webmanifest', 'zip', 'pdf', ...fontExts, 'xlsx', 'csv', 'mjs']
 };
 
 /**
@@ -626,32 +645,33 @@ AppBackend.prototype.start = function start(opts){
         // @ts-ignore : only for testing */
         this.getMainApp = function getMainApp(){ return mainApp; };
     }
-    this.shootDownBackend = function shootDownBackend(){
+    this.shutdownBackend = async function shutdownBackend(){
         console.log('shooting down:');
-        return new Promise(function(resolve,reject){
-            console.log('*','express server');
-            be.server.close(/** @param {Error} err */function(err){
-                if(err){
-                    console.log('*', err)
-                    reject(err);
-                }else{
-                    resolve();
-                }
-            });
-        }).then(async function(){
-            while(be.shootDownCallbackList.length){
-                var nextCallback=be.shootDownCallbackList.pop();
-                if(nextCallback!=null){
-                    console.log('*',nextCallback.message);
-                    await nextCallback.fun();
-                }
-            }
-        }).then(function(){
-            console.log('pg pool balance',pg.poolBalanceControl());
-            // @ts-ignore al hacer shootDown ya no hay mainApp. 
-            mainApp = null;
-            logWhy && logWhy();
-        });
+        var waitFor = [
+            new Promise(function(resolve,reject){
+                console.log('*','express server');
+                be.server.close(/** @param {Error} err */function(err){
+                    if(err){
+                        console.log('*', err)
+                        reject(err);
+                    }else{
+                        console.log('*','express server done!')
+                        resolve();
+                    }
+                });
+            }),
+            ...(be.shutdownCallbackList.map(x => (async function(){
+                console.log('shut:', x.message)
+                await x.fun()
+                console.log('done:', x.message)
+            })()))
+        ];
+        console.log('*', 'waiting for all')
+        await Promise.all(waitFor);
+        console.log('*', 'all done')
+        mainApp = null;
+        console.log('* logWhy',logWhy)
+        logWhy && logWhy();
     };
     return Promise.resolve().then(function(){
         var configList=be.configList();
@@ -682,10 +702,10 @@ AppBackend.prototype.start = function start(opts){
                 }
                 var sessionStoreName=be.config.server["session-store"];
                 if(sessionStoreName){
-                    if(config.devel && sessionStores[sessionStoreName+'Devel']){
+                    if(config.devel && be.sessionStores[sessionStoreName+'Devel']){
                         sessionStoreName+='Devel';
                     }
-                    var storeModule = sessionStores[sessionStoreName];
+                    var storeModule = be.sessionStores[sessionStoreName];
                     be.config.login.plus.store.module = storeModule;
                 }
                 be.config.install.dump.db.owner=coalesce(be.config.db.owner,be.config.install.dump.db.owner,be.config.db.user);
@@ -963,12 +983,18 @@ AppBackend.prototype.start = function start(opts){
                 loginPlusOpts.noLoggedUrlPath=be.config.login.unloggedLandPage;
             }
             mainApp.loginPlusManager.init(mainApp,changing(be.config.login.plus,loginPlusOpts));
-            be.shootDownCallbackList.push({
+            be.shutdownCallbackListAdd({
                 message:'login-plus manager',
                 fun:function(){
                     mainApp.loginPlusManager.closeManager();
                 }
             });
+            be.shutdownCallbackListAdd({
+                message:'pg',
+                fun:function(){
+                    pg.shutdown();
+                }
+            });
             mainApp.loginPlusManager.setValidatorStrategy(
                 function(req, username, password, done) {
                     var client;
@@ -1211,13 +1237,14 @@ AppBackend.prototype.checkDatabaseStructure = async function checkDatabaseStruct
         }
         if(be.config.login?.forget){
             try{
-                await client.query(`select tokentype, info, due from tokens limit 1`).fetchOneRowIfExists();
+                await client.query(`select tokentype, info, due from his.tokens limit 1`).fetchOneRowIfExists();
             }catch(err){
                 var mensaje = `
                     --------quizas falten los campos en la tabla tokens:
                     alter table tokens add column tokentype text;
                     alter table tokens add column info jsonb;
                     alter table tokens add column due timestamp;
+                    --------quizas falte moverla al esquema his.
                 `;
                 err.message += mensaje;
                 console.log(mensaje)
@@ -1528,17 +1555,17 @@ AppBackend.prototype.addProcedureServices = function addProcedureServices(forUnl
                                     });
                                 }else{
                                     if(progressInfo.lengthComputable){
-                                        if(new Date()-(context.lastMessageSended||0) > 500 || progressInfo.force){
-                                            context.lastMessageSended=new Date();
+                                        if (!context.lastMessageSended) {
+                                            context.lastMessageSended = {}
+                                        }
+                                        if(new Date()-(context.lastMessageSended[progressInfo.idGroup]||0) > 500 || progressInfo.force){
+                                            context.lastMessageSended[progressInfo.idGroup]=new Date();
                                             res.write(JSON.stringify({progress:progressInfo})+"\n");
                                             if(progressInfo.total){
                                                 var rate100=Math.floor(progressInfo.loaded*100/progressInfo.total);
                                                 var rate1000=Math.floor(progressInfo.loaded*1000/progressInfo.total);
-                                                if(rate100<1 && progressInfo.loaded>0){
-                                                    progress2send={message:'('+(rate1000||'½')+'‰)', ephemeral:true};
-                                                }else{
-                                                    progress2send={message:rate100+'%', ephemeral:true};
-                                                }
+                                                var message = rate100<1 && progressInfo.loaded>0 ? '('+(rate1000||'½')+'‰)' : rate100+'%';
+                                                progress2send={message, ephemeral:true, idGroup:progressInfo.idGroup};
                                             }
                                         }else{
                                             progress2send=null;
@@ -1741,14 +1768,26 @@ function require_resolve(moduleName){
     return resolved;
 }
 
-function resolve_module_dir(moduleName,path){
+function resolve_module_dir(moduleName,path,fileToCheck){
     var baseDir;
     if(packagejson.name==moduleName){
         baseDir=Path.join(process.cwd(), packagejson.main);
     }else{
         baseDir=require_resolve(moduleName);
     }
-    return Path.join(Path.dirname(baseDir),path||'');
+    var resultDir = Path.join(Path.dirname(baseDir),path||'');
+    if (fileToCheck) {
+        fs.stat(Path.join(resultDir, fileToCheck)).then(function(status){
+            if (!status) {
+                console.error('Error resolve_module_dir in:',resultDir,'no:',fileToCheck);
+            } else if(!status.isFile()) {
+                console.error('Error resolve_module_dir in:',resultDir,',',fileToCheck,'is not a file');
+            }
+        },function(err){
+            console.error('Error resolve_module_dir in:',resultDir,'err:',err);
+        });
+    }
+    return resultDir;
 }
 
 AppBackend.prototype.optsGenericForFiles = function optsGenericForFiles(req, opts){
@@ -1911,6 +1950,9 @@ AppBackend.prototype.addUnloggedServices = function addUnloggedServices(mainApp,
     // http://localhost:3033/img/login-logo-icon.png
     mainApp.get(Path.posix.join(baseUrl,'/img/login-logo-icon.png'), async function(req,res,next){
         var buscar = [
+            'unlogged/img/login-logo-icon.svg', 
+            'dist/unlogged/img/login-logo-icon.svg', 
+            'dist/client/unlogged/img/login-logo-icon.svg', 
             'unlogged/img/login-logo-icon.png', 
             'dist/unlogged/img/login-logo-icon.png', 
             'dist/client/unlogged/img/login-logo-icon.png', 
@@ -1938,7 +1980,7 @@ AppBackend.prototype.addUnloggedServices = function addUnloggedServices(mainApp,
         if(baseUrl=='/'){
             baseUrl='';
         }   
-        let baseLib = baseUrl + '/' + (moduleDef.path ? moduleDef.path : moduleDef.type == 'js'? 'lib': 'css');
+        let baseLib = baseUrl + '/' + (moduleDef.path ? moduleDef.path : be.esJavascript(moduleDef.type)? 'lib': 'css');
         try {
             var allowedExts=(moduleDef.type=='js'?['js','map']:[moduleDef.type]);
             mainApp.use(baseLib, serveContent(resolve_module_dir(moduleDef.module, moduleDef.modPath), { allowedExts }));
@@ -1952,7 +1994,7 @@ AppBackend.prototype.addUnloggedServices = function addUnloggedServices(mainApp,
                 }
             }
         } catch (error) {
-            console.error('ERROR: No se pudo servir el módulo ' + moduleDef.module);
+            console.error('ERROR: No se pudo servir el módulo', moduleDef.module);
             throw error;
         }
     })
@@ -2073,6 +2115,7 @@ AppBackend.prototype.clientIncludes = function clientIncludes(req, opts) {
 
 AppBackend.prototype.clientIncludesCompleted = function clientIncludesCompleted(req, opts) {
     let list = this.clientIncludes(req, opts);
+    var be = this;
     if(this.config.devel && this.config.devel.useFileDevelopment){
         list=list.map(includeDef=>{
             if(includeDef.fileProduction){
@@ -2088,14 +2131,15 @@ AppBackend.prototype.clientIncludesCompleted = function clientIncludesCompleted(
     return list.map(inclusion =>{
         var filename = inclusion.file? inclusion.file: (inclusion.module? Path.basename(require_resolve(inclusion.module,rPaths)): '');
         return changing({
-            src: inclusion.type == 'js' ? ((inclusion.path ? inclusion.path : 'lib') + '/' + (inclusion.file ? inclusion.file : filename)) : '',
+            src: be.esJavascript(inclusion.type) ? ((inclusion.path ? inclusion.path : 'lib') + '/' + (inclusion.file ? inclusion.file : filename)) : '',
             href: inclusion.type == 'css' ? ((inclusion.path ? inclusion.path : 'css') + '/' + (inclusion.file ? inclusion.file : (inclusion.module + '.css'))) : ''
         }, inclusion);
     });
 }
 
 AppBackend.prototype.clientModules = function clientModules(req, opts) {
-    return { scripts: this.clientIncludesCompleted(req, opts).filter(x => x.type == 'js').map(mod => {return { src: mod.src }})};
+    var be = this;
+    return { scripts: this.clientIncludesCompleted(req, opts).filter(x => be.esJavascript(x.type)).map(mod => {return { src: mod.src, type:mod.type=='mjs'?"module":null }})};
 }
 
 /**
@@ -2638,6 +2682,30 @@ AppBackend.prototype.dumpDbSchemaPartial = async function dumpDbSchemaPartial(pa
     }
     var schema=db.quoteIdent(be.config.db.schema);
     linesCreate.push("set role to "+owner+";");
+    linesCreate.push(`do 
+    $do$
+    declare
+      vExpectedVersion text := '${be.config.db["min-version"]}';
+      vCurrentVersion  text := (select setting from pg_settings where name = 'server_version');
+    begin
+      if jsonb('['||replace(vCurrentVersion,'.',',')||']') < jsonb('['||replace(vExpectedVersion,'.',',')||']') then
+        raise exception 'BACKEND-PLUS DUMP: Old PostrgreSQL Version % expected %', vCurrentVersion, vExpectedVersion;
+      end if;
+    end;
+    $do$;
+    `);
+    linesCreate.push(`do 
+    $do$
+    declare
+      vExpected text := 'UTF8';
+      vCurrent  text := coalesce((select setting from pg_settings where name = 'server_encoding'),'unknown');
+    begin
+      if vCurrent is distinct from vExpected then
+        raise exception 'BACKEND-PLUS DUMP: PostrgreSQL server_encoding %, expected %', vCurrent, vExpected;
+      end if;
+    end;
+    $do$;
+    `);
     linesCreate.push("drop schema if exists "+schema+' cascade;');
     if (be.config.install.dump["drop-his"]) {
         linesCreate.push("drop schema if exists his cascade;");
@@ -2796,6 +2864,9 @@ AppBackend.prototype.dumpDbSchemaPartial = async function dumpDbSchemaPartial(pa
                     case 'check': 
                         sql='check ('+cons.expr+')';
                     break;
+                    case 'exclude': 
+                        sql=`exclude using ${cons.using} (${cons.fields.map(f => typeof f == "string"?`${f} WITH =`:`${f.fieldName} WITH ${f.operator}`).join(', ')})`;
+                    break;
                     default:
                         throw new Error('constraintType not implemented: '+cons.constraintType);
                     }
@@ -2860,8 +2931,14 @@ AppBackend.prototype.dumpDbSchemaPartial = async function dumpDbSchemaPartial(pa
         if (err.code != 'ENOENT') throw err;
         try {
             var allTableContent = await fs.readFile('install/local-dump.psql','utf-8');
-            var startIndex = allTableContent.search(/\n-- Data for Name: /);
-            var lastIndex = allTableContent.search(/\nSELECT pg_catalog.setval/);
+            var startIndex = allTableContent.indexOf('-- Data for Name: ');
+            console.log('startIndex', startIndex);
+            var lastUseful = allTableContent.lastIndexOf('\nSELECT pg_catalog.setval') 
+            console.log('lastUseful', lastUseful);
+            if (lastUseful == -1) lastUseful = allTableContent.lastIndexOf('\n\\.\n');
+            console.log('lastUseful', lastUseful);
+            var lastIndex = allTableContent.indexOf('\n--', lastUseful);
+            console.log('lastIndex', lastIndex);
             allTableData = allTableContent.slice(startIndex, lastIndex);
         } catch(err) {
             if (err.code != 'ENOENT') throw err;
@@ -2945,22 +3022,32 @@ AppBackend.prototype.dumpDbSchemaPartial = async function dumpDbSchemaPartial(pa
                             lines[0][0]=lines[0][0].replace('\ufeff','')
                         }
                         rows=lines.slice(1);
+                        var filteredFieldDef = lines[0].filter(filterField);
                         if(tablesWithStrictSequence[tableName]){
                             var dataString="COPY "+db.quoteIdent(tableName)+" ("+
-                                lines[0].filter(filterField).map(db.quoteIdent).join(', ')+
+                                filteredFieldDef.map(db.quoteIdent).join(', ')+
                                 ') FROM stdin;\n'+
                                 rows.map(function(line){
-                                    return line.filter(function(_,i){ return filterField(lines[0][i]);}).map(function(value){
-                                        return value===''?'\\N':value.replace(/\\/g,'\\\\').replace(/\t/g,'\\t').replace(/\n/g,'\\n').replace(/\r/g,'\\r');
+                                    return line.filter(function(_,i){ return filterField(lines[0][i]);}).map(function(value,i){
+                                        var def = tableDef.field[filteredFieldDef[i]];
+                                        return value==='' ? (
+                                                def.allowEmptyText && ('nullable' in def) && !def.nullable ? '' : '\\N'
+                                            ): value.replace(/\\/g,'\\\\').replace(/\t/g,'\\t').replace(/\n/g,'\\n').replace(/\r/g,'\\r');
                                     }).join('\t')+'\n';
                                 }).join('')+'\\.\n';
                         }else{
                             var dataString="insert into "+db.quoteIdent(tableName)+" ("+
-                                lines[0].filter(filterField).map(db.quoteIdent).join(', ')+
+                                filteredFieldDef.map(db.quoteIdent).join(', ')+
                                 ') values\n'+
                                 rows.map(function(line){
-                                    return "("+line.filter(function(_,i){ return filterField(lines[0][i]);}).map(function(value){
-                                        return value===''?'null':db.quoteNullable(value);
+                                    return "("+line.filter(function(_,i){ return filterField(lines[0][i]);}).map(function(value,i){
+                                        var def = tableDef.field[filteredFieldDef[i]];
+                                        if(def == null) {
+                                            throw Error("no se encuentra la columna "+filteredFieldDef[i]+" en "+tableName);
+                                        }
+                                        return value==='' ? (
+                                            def.allowEmptyText && ('nullable' in def) && !def.nullable ? "''" : 'null' 
+                                        ) : db.quoteNullable(value);
                                     }).join(', ')+")";
                                 }).join(',\n')+';\n';
                         }
@@ -2985,27 +3072,33 @@ AppBackend.prototype.dumpDbSchemaPartial = async function dumpDbSchemaPartial(pa
     }
     let installFolders = be.config.install.dump.folders ?? ['install']
     let texts = await Promise.all(
-        ['prepare.sql','pre-adapt.sql','adapt.sql']
-        .concat(be.config.install.dump.scripts['prepare'])
-        .concat(be.config.install.dump.scripts['post-adapt'])
-        .map(async function(fileName){
-            if (!fileName) return '';
+        [
+            ['prepare.sql'],
+            ['pre-adapt.sql'].concat(be.config.install.dump.scripts['pre-adapt']),
+            ['adapt.sql'],
+            be.config.install.dump.scripts['prepare'] ?? [],
+            be.config.install.dump.scripts['post-adapt'] ?? []
+        ]
+        .map(async function(fileNames){
+            if (!fileNames) return '';
             var i = 0; 
-            var content;
-            do {
-                var folder = installFolders[i];
-                try{
-                    content = await fs.readFile(be.rootPath+'/'+folder+'/'+fileName, {encoding:'UTF8'});
-                } catch (err) {
-                    if(err.code!='ENOENT') throw err;
-                }
-                i++;
-            } while (i < installFolders.length && !content);
-            if (!content) {
-                return '-- no '+fileName+'\n';
-            } else {
-                return '-- '+folder+'/'+fileName+'\n'+content;
-            };
+            return (await Promise.all(fileNames.map(async fileName => {
+                var content;
+                do {
+                    var folder = installFolders[i];
+                    try{
+                        content = await fs.readFile(be.rootPath+'/'+folder+'/'+fileName, {encoding:'UTF8'});
+                    } catch (err) {
+                        if(err.code!='ENOENT') throw err;
+                    }
+                    i++;
+                } while (i < installFolders.length && !content);
+                if (!content) {
+                    return '-- no '+fileName+'\n';
+                } else {
+                    return '-- '+folder+'/'+fileName+'\n'+content;
+                };
+            }))).join('\n')
         })
     );
 
@@ -3035,7 +3128,7 @@ AppBackend.prototype.dumpDbSchemaPartial = async function dumpDbSchemaPartial(pa
         '\n-- common'+
         common+'\n'+
         (complete? '\n\n--prepare.sql\n'+ texts[0]+'\n\n' :'' )+
-        (complete? texts.slice(3,3+prepareList.length).join('\n\n')+'\n\n' : '' )+
+        (complete? texts[3] + '\n\n' : '' )+
         '\n-- functions\n' + functionLines.join('\n')+
         '\n-- lines \n' + lines.join('\n')+
         (complete? ('\n\n-- pre-ADAPTs\n'+texts[1]+'\n\n') : '' )+
@@ -3045,7 +3138,7 @@ AppBackend.prototype.dumpDbSchemaPartial = async function dumpDbSchemaPartial(pa
         '\n-- FKs\n' + fkLines.join('\n')+
         '\n-- index\n' + indexLines.join('\n')+
         '\n-- policies\n' + policyLines.join('\n')+
-        (complete? texts.slice(3+prepareList.length).join('\n\n')+'\n\n' : '' )+
+        (complete? texts[4] + '\n\n' : '' )+
         (complete? (be.config.install.dump.enances==='inline'?enancePart:'') :'')
     ).replace(/\uFEFF/g /*inner BOM replacing*/,'\n\n').replace(
         new RegExp(escapeRegExp(db.quoteIdent(be.config.install.dump.db["owner4special-scripts"])),'g'),

package/lib/procedures-table.js

@@ -201,7 +201,7 @@ ProcedureTables = [
                 tables:{},
                 tableNames:{}
             };
-            var expectedMainPrimaryKeyValues=parameters.primaryKeyValues;
+            var expectedMainPrimaryKeyValues = parameters.primaryKeyValues;
             /** @type { TableUpdater } */
             // @ts-expect-error parece no asignada pero al menos un nombre hay en el ciclo y por lo tanto va a tener valor
             var updateTarget = {}
@@ -257,6 +257,7 @@ ProcedureTables = [
             });
             var result;
             var mainResult;
+            var initialUpdatesLength = updates.length;
             while(updates.length){
                 var {fieldNames, values, defTable, main} = notnull(updates.shift());
                 var values4Insert = values.slice(0);
@@ -272,7 +273,7 @@ ProcedureTables = [
                             values4Insert[i] = value;
                         }
                     }
-                    if(defField.generatedAs){
+                    if(defField.generatedAs || defField.inTable === false){
                         var i = fieldNames4Insert.indexOf(defField.name);
                         if(i!==-1){
                             fieldNames4Insert.splice(i,1);
@@ -284,10 +285,11 @@ ProcedureTables = [
                 var primaryKeyValues=[]
                 var primaryKeyValuesForUpdate=[]
                 var primaryKeyValueObject={};
+                var fieldNames4InsertWithoutPkAddings = fieldNames4Insert.slice(0);
                 defTable.primaryKey.forEach(function(name,i){
                     var value = main?parameters.primaryKeyValues[i]:coalesce(parameters.newRow[name],parameters.oldRow[name],undefined);
+                    var defField = defTable.field[name];
                     if(value===undefined){
-                        var defField = defTable.field[name];
                         value = defTable.prefilledField && name in defTable.prefilledField ? defTable.prefilledField[name] : (
                             'defaultValue' in defField?defField.defaultValue:(
                             'specialDefaultValue' in defField?new Error("my.specialDefaultValue[defField.specialDefaultValue](name, {row:{}}, {row:previousRow})"):undefined
@@ -304,13 +306,16 @@ ProcedureTables = [
                     }else{
                         primaryKeyValuesForUpdate[i]=value;
                     }
-                    if(!fieldNames4Insert.includes(name) && value != null){
+                    if(!fieldNames4Insert.includes(name) && value != null && defField.inTable !== false){
                         fieldNames4Insert.push(name); 
                         values4Insert.push(value);
                     }
                 });
-                if(main){
-                    expectedMainPrimaryKeyValues=primaryKeyValues;
+                if (main) {
+                    expectedMainPrimaryKeyValues = primaryKeyValues;
+                } else if (defTable.sql.setExpectedPkValues) {
+                    expectedMainPrimaryKeyValues = primaryKeyValues.slice(0);
+                    while (expectedMainPrimaryKeyValues.length < mainDefTable.primaryKey.length) expectedMainPrimaryKeyValues.push(null);
                 }
                 var returningClausule='';
                 if(opts && opts.forImport){
@@ -331,7 +336,9 @@ ProcedureTables = [
                     return result;
                 }
                 var sql;
-                if(defTable && parameters.status=='new'){
+                if (defTable?.sql?.isTable === false && initialUpdatesLength > 1 && fieldNames4InsertWithoutPkAddings.length == 0) {
+                    // New feature. Ignore attempts to save to non tables when pseudo tables save in others
+                } else if(defTable && parameters.status=='new'){
                     result = await insertFun();
                 }else if(defTable && primaryKeyValues.length==defTable.primaryKey.length){
                     var values4Update = [];
@@ -400,6 +407,12 @@ ProcedureTables = [
                 var rowFrom = parameters.status!='new' && !plainUpdate.values.length?
                     likeAr.toPlainObject(mainDefTable.primaryKey,expectedMainPrimaryKeyValues):
                     (mainResult||result).row;
+                for (var fieldname of mainDefTable.primaryKey) {
+                    if (rowFrom[fieldname] == null) {
+                        var newValue = parameters.newRow[fieldname] ?? parameters.oldRow[fieldname];
+                        if (newValue != null) rowFrom[fieldname] = newValue
+                    }
+                }
                 result = await be.queryValuesOfUniqueRow(context, mainDefTable, rowFrom);
             }
             return {command:insertOrUpdate, row:result.row};
@@ -417,16 +430,26 @@ ProcedureTables = [
             if(!defTable.allow.delete){
                 throw changing(new Error("Deletes not allowed"),{status:403});
             }
-            var primaryKeyFields=defTable.primaryKey4Delete||defTable.primaryKey;
-            var primaryKeyValues=parameters.primaryKeyValues;
+            if (defTable.primaryKey4Delete) {
+                console.log("DEPRECATED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! defTable.primaryKey4Delete", defTable.name);
+            }
+            var primaryKeyFields = defTable.primaryKey4Delete ?? defTable.sql.primaryKey4Delete ?? defTable.primaryKey;
+            var primaryKeyValues = [];
+            var tableName4Delete = defTable.sql.tableName4Delete ?? defTable.sql.tableName;
+            for (var fieldName of primaryKeyFields) {
+                var pos = defTable.primaryKey.indexOf(fieldName);
+                if (pos > -1) {
+                    primaryKeyValues.push(parameters.primaryKeyValues[pos]);
+                }
+            }
             if(defTable && primaryKeyValues.length==primaryKeyFields.length){
                 var sqlDelete;
                 if(defTable.sql.logicalDeletes){
-                    sqlDelete="UPDATE "+be.db.quoteIdent(defTable.sql.tableName)+
+                    sqlDelete="UPDATE "+be.db.quoteIdent(tableName4Delete)+
                     " SET "+be.db.quoteIdent(defTable.sql.logicalDeletes.fieldName)+
                     " = "+be.db.quoteNullable(defTable.sql.logicalDeletes.valueToDelete);
                 }else{
-                    sqlDelete="DELETE FROM "+be.db.quoteIdent(defTable.sql.tableName);
+                    sqlDelete="DELETE FROM "+be.db.quoteIdent(tableName4Delete);
                 }
                 sqlDelete+=" WHERE "+primaryKeyFields.map(function(fieldName, i){
                     return be.db.quoteIdent(fieldName)+" = $"+(i+1);
@@ -453,6 +476,7 @@ ProcedureTables = [
             context.informProgress({message:be.messages.server.deleting})
             var be=context.be;
             var defTable=be.tableStructures[parameters.table](context);
+            var tableName4Delete = defTable.sql.tableName4Delete ?? defTable.sql.tableName;
             if(!defTable.allow.delete && !defTable.allow.deleteAll){
                 throw changing(new Error("Deletes not allowed"),{status:403});
             }
@@ -480,7 +504,7 @@ ProcedureTables = [
                     }
                 });
                 return context.client.query(
-                    "DELETE FROM "+be.db.quoteIdent(defTable.sql.tableName)+
+                    "DELETE FROM "+be.db.quoteIdent(tableName4Delete)+
                     " WHERE "+whereParts.join(" AND ")+" RETURNING 1",
                     dataParams
                 ).fetchUniqueRow().then(function(){
@@ -490,7 +514,7 @@ ProcedureTables = [
             };
             var result = await Promise.all(parameters.rowsToDelete.map(deleteOneRow)).then(function(){
                 return context.client.query(
-                    "SELECT count(*) remaining_record_count FROM "+be.db.quoteIdent(defTable.sql.tableName)
+                    "SELECT count(*) remaining_record_count FROM "+be.db.quoteIdent(tableName4Delete)
                 ).fetchUniqueRow().then(function(result){
                     context.informProgress({lengthComputable:true, loaded:deleteCounter, total:parameters.rowsToDelete.length, force:true});
                     return result.row;
@@ -693,7 +717,7 @@ ProcedureTables = [
             {name: 'replaceNewLineWithSpace'},
         ],
         files:{count:1},
-        coreFunction:async function(context, parameters, files){
+        coreFunction:async function table_upload(context, parameters, files){
             var be=context.be;
             var doing="opening file";
             var skipUnknownFieldsAtImport = coalesce(parameters.skipUnknownFieldsAtImport, be.config.skipUnknownFieldsAtImport);
@@ -823,6 +847,9 @@ ProcedureTables = [
                                     if(value != null && parameters.skipUnknownFieldsAtImport){
                                         value = value.replace(/(\s|\u00A0)+/g,' ').trim()
                                     }
+                                    if (value == null && field.allowEmptyText && ('nullable' in field) && !field.nullable ) {
+                                        value = ''
+                                    }
                                     newRow[field.name]=value;
                                 }
                             });
@@ -902,6 +929,9 @@ ProcedureTables = [
                                     if(Number.isNaN(value)){
                                         value=null;
                                     }
+                                    if (value == null && field.allowEmptyText && ('nullable' in field) && !field.nullable ) {
+                                        value = ''
+                                    }
                                     if(field.defaultForOtherFields){
                                         addFieldToOthers(othersArray, field, value);
                                     }else{