@xuda.io/xuda-dbs-plugin-xuda 1.0.318 → 1.0.320

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xuda.io/xuda-dbs-plugin-xuda",
-  "version": "1.0.318",
+  "version": "1.0.320",
   "description": "Xuda Database Socket for Xuda's proprietary structure powered by CouchDB",
   "scripts": {
     "pub": "npm version patch --force && npm publish --access public"

package/server.mjs

@@ -515,27 +515,68 @@ const query_db = async function (e, db, app_id_reference, table_obj) {
       return raw_data();
     };
 
+    const get_mango_index_fields = function (selector) {
+      const fields = new Set();
+      const add_selector_fields = function (obj) {
+        if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+          return;
+        }
+        for (const [key, val] of Object.entries(obj)) {
+          if (['$and', '$or', '$nor'].includes(key)) {
+            if (Array.isArray(val)) {
+              val.forEach(add_selector_fields);
+            }
+            continue;
+          }
+          if (key.startsWith('$')) {
+            continue;
+          }
+          fields.add(key);
+        }
+      };
+      add_selector_fields(selector);
+      return [...fields];
+    };
+
+    const create_mango_index_for_selector = async function () {
+      const index = get_mango_index_fields(opt.selector);
+      if (!index.length) {
+        return;
+      }
+      const index_suffix = index.join('_').replace(/[^a-zA-Z0-9_]+/g, '_').slice(0, 80) || 'selector';
+      const mango_index_obj = {
+        index: {
+          fields: index,
+        },
+        name: `index_${e.table_id}_${index_suffix}`,
+        ddoc: `mango_index_table_${e.table_id}_${index_suffix}`,
+      };
+      try {
+        await db.createIndex(mango_index_obj);
+      } catch (err) {
+        // The read already succeeded; index creation is only an optimization.
+      }
+    };
+
+    // A COUNT must span the WHOLE result set, not the current page. `opt` carries
+    // the page limit/skip, so counting body.docs.length under them could never
+    // exceed one page. Ask for ids only (unless count_data's index-grouping
+    // branch needs the field data) and cap the scan so a pathological table
+    // cannot stall the request.
+    const COUNT_SCAN_CAP = 100000;
+    const find_opt = e.count
+      ? Object.assign({}, opt, {
+          limit: COUNT_SCAN_CAP,
+          skip: 0,
+          fields: e.indexId ? opt.fields : ['_id'],
+        })
+      : opt;
+
     try {
       try {
-        const doc = await db.find(opt);
-        var mango_index_obj;
+        const doc = await db.find(find_opt);
         if (doc?.warning?.includes('No matching index found')) {
-          const index_name = `index_${e.table_id}_${new Date().valueOf().toString()}`;
-          var index = [];
-
-          for (const [key, val] of Object.entries(opt.selector)) {
-            index.push(key);
-          }
-          mango_index_obj = {
-            index: {
-              fields: index,
-            },
-            name: index_name,
-            ddoc: `mango_index_table_${e.table_id}`,
-          };
-          db.createIndex(mango_index_obj).then((result) => {
-            console.log(result);
-          });
+          await create_mango_index_for_selector();
         }
         return await done(doc);
       } catch (err) {
@@ -544,7 +585,7 @@ const query_db = async function (e, db, app_id_reference, table_obj) {
           const _selector = _.cloneDeep(opt.selector);
           delete _selector['$and'];
           const _sort = _.cloneDeep(opt.sort);
-          mango_index_obj = {
+          const mango_index_obj = {
             index: {
               fields: Object.keys(
                 _.reduce(
@@ -564,7 +605,7 @@ const query_db = async function (e, db, app_id_reference, table_obj) {
           try {
             const result = await db.createIndex(mango_index_obj);
 
-            const doc = await db.find(opt);
+            const doc = await db.find(find_opt);
             return await done(doc);
           } catch (err) {
             return { code: -1, data: err.message };
@@ -692,7 +733,31 @@ const query_db = async function (e, db, app_id_reference, table_obj) {
     return await runtime_get_query_data();
   }
 
-  if (e.count && !e.filter_from) {
+  // A COUNT must answer "how many rows match THIS query", not "how many rows are
+  // in the table". count_tables() reads the db_table_counts reduce view keyed by
+  // table_id alone, so it ignores the selector completely — and `filter_from` is
+  // only ever set for dataSourceFilterModelType==='index', so every MONGO-filtered
+  // count landed here and came back with the whole-table total. Live symptom: a
+  // filtered question list returned its correct rows while rows_found stayed 837,
+  // so the pager kept offering 168 pages over an empty/short result set.
+  // Keep the cheap view for a genuinely UNFILTERED count; anything carrying a
+  // selector is counted through Mango against that same selector.
+  const _has_filter_value = function (v) {
+    if (v === undefined || v === null || v === '') return false;
+    try {
+      const o = typeof v === 'string' ? JSON.parse(v) : v;
+      return !_.isEmpty(o);
+    } catch (err) {
+      return true; // present but unparseable — assume it filters; never over-count
+    }
+  };
+  const _count_is_filtered =
+    !!e.filter_from ||
+    _has_filter_value(e.filterModelMongo) ||
+    _has_filter_value(e.filterModelUserMongo) ||
+    _has_filter_value(e.filterModelNative);
+
+  if (e.count && !_count_is_filtered) {
     return await count_tables();
   }
 

package/studio.mjs

@@ -180,6 +180,20 @@ const get_cast_val = async function (source, attributeP, typeP, valP) {
   return xu_cast(typeP, valP, success, fail);
 };
 
+const get_json_value = function (value, fallback) {
+  if (typeof value === 'undefined' || value === null || value === '') {
+    return fallback;
+  }
+  if (typeof value === 'string') {
+    try {
+      return JSON.parse(value);
+    } catch (err) {
+      return fallback;
+    }
+  }
+  return value;
+};
+
 const get_opt = function (e, table_obj) {
   var limit = 99999;
   var skip = 0;
@@ -244,9 +258,9 @@ const get_opt = function (e, table_obj) {
     opt.sort = sort;
   }
 
-  let _sortModel = e?.sortModel || [];
-  if (typeof _sortModel === 'string') {
-    _sortModel = JSON.parse(e.sortModel);
+  let _sortModel = get_json_value(e?.sortModel, []);
+  if (!Array.isArray(_sortModel)) {
+    _sortModel = [];
   }
 
   if (_sortModel.length) {
@@ -288,7 +302,7 @@ const get_opt = function (e, table_obj) {
     opt.fields[key] = 'udfData.data.' + val;
   }
 
-  if (!e?.sortModel || !JSON.parse(e.sortModel).length) {
+  if (!e?.sortModel || !_sortModel.length) {
     // added 2021 09 10
     if (opt.sort) {
       for (const [key, val] of Object.entries(opt.sort)) {
@@ -372,14 +386,16 @@ const get_opt = function (e, table_obj) {
     return recursiveReplace(query);
   }
 
-  if (e.dataSourceFilterModelType === 'query' && e.filterModelMongo) {
-    selector_new['$and'] = [replaceRegexOptions(replaceKeysInQuery(JSON.parse(e.filterModelMongo)))];
+  const filterModelMongo = get_json_value(e.filterModelMongo, {});
+  if (e.dataSourceFilterModelType === 'query' && !_.isEmpty(filterModelMongo)) {
+    selector_new['$and'] = [replaceRegexOptions(replaceKeysInQuery(filterModelMongo))];
   }
-  if (e.filterModelUserMongo) {
+  const filterModelUserMongo = get_json_value(e.filterModelUserMongo, {});
+  if (!_.isEmpty(filterModelUserMongo)) {
     if (!selector_new['$and']) {
       selector_new['$and'] = [];
     }
-    selector_new['$and'].push(replaceRegexOptions(replaceKeysInQuery(JSON.parse(e.filterModelUserMongo))));
+    selector_new['$and'].push(replaceRegexOptions(replaceKeysInQuery(filterModelUserMongo)));
   }
 
   if (e.dataSourceFilterModelType === 'index') {
@@ -587,44 +603,66 @@ const query_db = async function (e, db, table_obj) {
     };
 
     // xuda
+    const get_mango_index_fields = function (selector) {
+      const fields = new Set();
+      const add_selector_fields = function (obj) {
+        if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+          return;
+        }
+        for (const [key, val] of Object.entries(obj)) {
+          if (['$and', '$or', '$nor'].includes(key)) {
+            if (Array.isArray(val)) {
+              val.forEach(add_selector_fields);
+            }
+            continue;
+          }
+          if (key.startsWith('$')) {
+            continue;
+          }
+          fields.add(key);
+        }
+      };
+      add_selector_fields(selector);
+      return [...fields];
+    };
+
+    const create_mango_index_for_selector = async function () {
+      const index = get_mango_index_fields(opt.selector);
+      if (!index.length) {
+        return;
+      }
+      const index_suffix = index.join('_').replace(/[^a-zA-Z0-9_]+/g, '_').slice(0, 80) || 'selector';
+      const mango_index_obj = {
+        index: {
+          fields: index,
+        },
+        name: `index_${e.table_id}_${index_suffix}`,
+        ddoc: `mango_index_table_${e.table_id}_${index_suffix}`,
+      };
+      try {
+        await db.createIndex(mango_index_obj);
+      } catch (err) {
+        // The read already succeeded; index creation is only an optimization.
+      }
+    };
 
     try {
       try {
         // console.log("opt", opt);
 
         const doc = await db.find(opt);
-        var mango_index_obj;
         if (doc?.warning?.includes('No matching index found')) {
-          try {
-            const index_name = `index_${e.table_id}_${new Date().valueOf().toString()}`;
-            var index = [];
-
-            for (const [key, val] of Object.entries(opt.selector)) {
-              index.push(key);
-            }
-            mango_index_obj = {
-              index: {
-                fields: index,
-              },
-              name: index_name,
-              ddoc: `mango_index_table_${e.table_id}`,
-            };
-            const result = await db.createIndex(mango_index_obj);
-            // console.log("createIndex", result);
-            throw new Error('creating index in progress');
-          } catch (err) {
-            return { code: -1, data: err.message };
-          }
+          await create_mango_index_for_selector();
         }
         return await done(doc);
       } catch (err) {
-        if (err.message.includes('Cannot sort on field')) {
+        if (err?.message?.includes('Cannot sort on field')) {
           const index_name = `index_${e.table_id}_${new Date().valueOf().toString()}`;
 
           // const _selector = _.cloneDeep(opt.selector);
           let _sort = _.cloneDeep(opt.sort);
 
-          mango_index_obj = {
+          const mango_index_obj = {
             index: {
               fields: [],
             },
@@ -655,32 +693,17 @@ const query_db = async function (e, db, table_obj) {
 
           try {
             const result = await db.createIndex(mango_index_obj);
-            // const monitor_indexing = async function () {
-            //   return new Promise((resolve, reject) => {
-            //     db.on("indexing", function (event) {
-            //       // called when indexes are updated
-            //       console.log("indexes event", event);
-            //       // resolve();
-            //     });
-            //     setTimeout(() => {
-            //       resolve();
-            //     }, 5000);
-            //   });
-            // };
-            // await monitor_indexing();
-            // throw new Error({ code: -88, data: "creating index" });
-            throw new Error('creating index in progress');
-            // const doc = await db.find(opt);
-            // return await done(doc);
+            const doc = await db.find(opt);
+            return await done(doc);
           } catch (err) {
             return { code: -1, data: err.message };
           }
         } else {
-          return { code: -1, data: err.message };
+          return { code: -1, data: err?.message || err };
         }
       }
     } catch (err) {
-      return { code: -1, data: err.message };
+      return { code: -1, data: err?.message || err };
     }
   };