@tiledesk/tiledesk-server 2.19.11 → 2.19.13

package/services/dataTableService.js

@@ -0,0 +1,545 @@
+const { Table, TableRow } = require('../models/dataTable');
+const dataTableUtiles = require('../utils/dataTableUtils');
+const { calculateBsonSize } = require('../utils/dataTableBsonUtils');
+const mongoose = require('mongoose');
+
+const DEFAULT_TABLE_MAX_SIZE_BYTES = 30 * 1024 * 1024; // 30 MB
+
+function getTableMaxSizeBytes() {
+  const val = process.env.TABLE_MAX_SIZE_BYTES || DEFAULT_TABLE_MAX_SIZE_BYTES;
+  const n = parseInt(val, 10);
+  return isNaN(n) || n <= 0 ? DEFAULT_TABLE_MAX_SIZE_BYTES : n;
+}
+
+function getTableStats(table) {
+  if (!table) {
+    return { rows: 0, sizeBytes: 0 };
+  }
+  let stats;
+  if (typeof table.get === 'function') {
+    stats = table.get('stats');
+  } else {
+    stats = table.stats;
+  }
+  stats = stats || {};
+  return {
+    rows: stats.rows || 0,
+    sizeBytes: stats.sizeBytes || 0,
+  };
+}
+
+function assertTableSizeLimit(table, delta) {
+  if (delta <= 0) return;
+  const maxSize = getTableMaxSizeBytes();
+  const stats = getTableStats(table);
+  if (stats.sizeBytes + delta > maxSize) {
+    throw new Error('TABLE_SIZE_LIMIT_EXCEEDED');
+  }
+}
+
+function rowToPlainObject(row) {
+  return row.toObject ? row.toObject({ depopulate: true }) : row;
+}
+
+function buildInsertRowDoc(id_project, id_table, data, id_row) {
+  const now = new Date();
+  return {
+    _id: id_row || new mongoose.Types.ObjectId(),
+    id_project: id_project,
+    id_table: id_table,
+    data: data,
+    createdAt: now,
+    updatedAt: now,
+  };
+}
+
+function buildUpdatedRowDoc(existingRow, dataPatch) {
+  const plain = rowToPlainObject(existingRow);
+  plain.data = Object.assign({}, plain.data || {}, dataPatch);
+  plain.updatedAt = new Date();
+  return plain;
+}
+
+function dataPatchFromUpdate(update) {
+  const data = {};
+  const set = (update && update.$set) || {};
+  for (const key in set) {
+    if (Object.prototype.hasOwnProperty.call(set, key) && key.indexOf('data.') === 0) {
+      data[key.slice(5)] = set[key];
+    }
+  }
+  return data;
+}
+
+async function incrementTableStats(id_project, id_table, rowsDelta, sizeDelta) {
+  if (rowsDelta === 0 && sizeDelta === 0) return;
+  await Table.findOneAndUpdate(
+    { id_project: id_project, _id: id_table },
+    { $inc: { 'stats.rows': rowsDelta, 'stats.sizeBytes': sizeDelta } }
+  );
+}
+
+function computeInsertDelta(rowDoc) {
+  return calculateBsonSize(rowDoc);
+}
+
+function computeUpdateDelta(existingRows, update) {
+  const dataPatch = dataPatchFromUpdate(update);
+  let totalDelta = 0;
+  for (let i = 0; i < existingRows.length; i++) {
+    const oldSize = calculateBsonSize(rowToPlainObject(existingRows[i]));
+    const newSize = calculateBsonSize(buildUpdatedRowDoc(existingRows[i], dataPatch));
+    totalDelta += newSize - oldSize;
+  }
+  return totalDelta;
+}
+
+function computeDeleteDelta(existingRows) {
+  let totalSize = 0;
+  for (let i = 0; i < existingRows.length; i++) {
+    totalSize += calculateBsonSize(rowToPlainObject(existingRows[i]));
+  }
+  return { rowsDelta: -existingRows.length, sizeDelta: -totalSize };
+}
+
+function formatTable(table) {
+  const o = table.toObject ? table.toObject() : table;
+  return {
+    _id: o._id,
+    id_project: o.id_project,
+    name: o.name,
+    schema: o.schema || [],
+    stats: getTableStats(o),
+    createdAt: o.createdAt,
+    updatedAt: o.updatedAt,
+  };
+}
+
+function formatRow(row, schema) {
+  const o = row.toObject ? row.toObject() : row;
+  return {
+    _id: o._id,
+    id_project: o.id_project,
+    id_table: o.id_table,
+    data: dataTableUtiles.fillRowDataFromSchema(o.data, schema || []),
+    createdAt: o.createdAt,
+    updatedAt: o.updatedAt,
+  };
+}
+
+function formatRowForList(row, schema) {
+  const o = row.toObject ? row.toObject() : row;
+  return Object.assign(
+    { _id: o._id },
+    dataTableUtiles.fillRowDataFromSchema(o.data, schema || [])
+  );
+}
+
+function hasConditions(body) {
+  return !!(body.conditions && body.conditions.length && !body.id_row);
+}
+
+class DataTableService {
+
+  async listTables(id_project) {
+    const tables = await Table.find({ id_project }).sort({ createdAt: -1 });
+    return tables.map(formatTable);
+  }
+
+  async getTable(id_project, id_table) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const result = formatTable(table);
+    const rows = await TableRow.find({ id_project: id_project, id_table: id_table })
+      .sort({ createdAt: 1 })
+      .limit(100);
+    const schema = table.schema || [];
+    result.rows = rows.map(function (row) {
+      return dataTableUtiles.fillRowDataFromSchema(row.data, schema);
+    });
+    return result;
+  }
+
+  async createTable(id_project, body, createdBy) {
+    if (!body.name || typeof body.name !== 'string' || !body.name.trim()) {
+      throw new Error('name is required');
+    }
+    const schema = dataTableUtiles.normalizeSchemaForCreate(body.schema);
+    const table = await Table.create({
+      id_project: id_project,
+      name: body.name.trim(),
+      schema: schema,
+      createdBy: createdBy,
+    });
+    return formatTable(table);
+  }
+
+  async updateTable(id_project, id_table, body) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const update = {};
+    if (body.name !== undefined) {
+      if (!body.name || typeof body.name !== 'string' || !body.name.trim()) {
+        throw new Error('name is required');
+      }
+      update.name = body.name.trim();
+    }
+
+    if (body.schema === undefined) {
+      if (Object.keys(update).length === 0) {
+        throw new Error('name or schema is required');
+      }
+      const renamedOnly = await Table.findOneAndUpdate(
+        { id_project: id_project, _id: id_table },
+        update,
+        { new: true }
+      );
+      return formatTable(renamedOnly);
+    }
+
+    const currentSchema = table.schema || [];
+    const resolved = dataTableUtiles.resolveSchemaUpdate(body.schema, currentSchema);
+
+    for (let d = 0; d < resolved.deletes.length; d++) {
+      const delName = resolved.deletes[d];
+      await TableRow.updateMany(
+        { id_project: id_project, id_table: id_table },
+        { $unset: { ['data.' + delName]: '' } }
+      );
+    }
+
+    for (let r = 0; r < resolved.renames.length; r++) {
+      const ren = resolved.renames[r];
+      await TableRow.updateMany(
+        { id_project: id_project, id_table: id_table },
+        { $rename: { ['data.' + ren.oldName]: 'data.' + ren.newName } }
+      );
+    }
+
+    update.schema = resolved.schema;
+    const updated = await Table.findOneAndUpdate(
+      { id_project: id_project, _id: id_table },
+      update,
+      { new: true }
+    );
+    return formatTable(updated);
+  }
+
+  async deleteTable(id_project, id_table) {
+    const table = await Table.findOneAndDelete({ id_project: id_project, _id: id_table });
+    if (!table) return false;
+    await TableRow.deleteMany({ id_project: id_project, id_table: id_table });
+    return true;
+  }
+
+  async addColumn(id_project, id_table, columnInput) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const schema = table.schema || [];
+    const newCol = dataTableUtiles.normalizeColumnToAdd(columnInput, schema.length, schema);
+    schema.push(newCol);
+
+    const updated = await Table.findOneAndUpdate(
+      { id_project: id_project, _id: id_table },
+      { schema: schema },
+      { new: true }
+    );
+    return formatTable(updated);
+  }
+
+  async renameColumn(id_project, id_table, columnId, newName) {
+    if (!newName || typeof newName !== 'string' || !newName.trim()) {
+      throw new Error('name is required');
+    }
+    const trimmed = newName.trim();
+
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const schema = table.schema || [];
+    const col = dataTableUtiles.findColumn(schema, columnId);
+    if (!col) throw new Error('Column not found');
+
+    if (col.name === trimmed) return formatTable(table);
+
+    if (dataTableUtiles.findColumnByName(schema, trimmed)) {
+      throw new Error('Column name already exists: ' + trimmed);
+    }
+
+    const oldName = col.name;
+    const newSchema = schema.map(function (c) {
+      if (c.id === columnId) return { id: c.id, name: trimmed, type: c.type, index: c.index };
+      return c;
+    });
+
+    await TableRow.updateMany(
+      { id_project: id_project, id_table: id_table },
+      { $rename: { ['data.' + oldName]: 'data.' + trimmed } }
+    );
+
+    const updated = await Table.findOneAndUpdate(
+      { id_project: id_project, _id: id_table },
+      { schema: newSchema },
+      { new: true }
+    );
+    return formatTable(updated);
+  }
+
+  async deleteColumn(id_project, id_table, columnId) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const schema = table.schema || [];
+    const col = dataTableUtiles.findColumn(schema, columnId);
+    if (!col) throw new Error('Column not found');
+
+    const newSchema = schema.filter(function (c) { return c.id !== columnId; });
+
+    await TableRow.updateMany(
+      { id_project: id_project, id_table: id_table },
+      { $unset: { ['data.' + col.name]: '' } }
+    );
+
+    const updated = await Table.findOneAndUpdate(
+      { id_project: id_project, _id: id_table },
+      { schema: newSchema },
+      { new: true }
+    );
+    return formatTable(updated);
+  }
+
+  async listRows(id_project, id_table) {
+    return this.listRowsWithFilter(id_project, id_table, {});
+  }
+
+  async listRowsWithFilter(id_project, id_table, params) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const schema = table.schema || [];
+    let conditions = params.conditions;
+
+    if (typeof conditions === 'string') {
+      try {
+        conditions = JSON.parse(conditions);
+      } catch (e) {
+        throw new Error('conditions must be a valid JSON array');
+      }
+    }
+
+    const baseQuery = { id_project: id_project, id_table: id_table };
+
+    if (conditions && Array.isArray(conditions) && conditions.length > 0) {
+      const filter = dataTableUtiles.buildSearchFilter({
+        must_match: params.must_match,
+        match: params.match,
+        conditions: conditions,
+      }, schema);
+      Object.assign(baseQuery, filter);
+    }
+
+    const sortOrder = params.sort === -1 || params.sort === 'desc' ? -1 : 1;
+    let query = TableRow.find(baseQuery).sort({ createdAt: sortOrder });
+    if (params.limit != null) {
+      const limit = parseInt(params.limit, 10);
+      if (!isNaN(limit) && limit > 0) {
+        query = query.limit(limit);
+      }
+    }
+    const rows = await query;
+    return rows.map(function (row) {
+      return formatRowForList(row, schema);
+    });
+  }
+
+  async getRow(id_project, id_table, id_row) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+    const row = await TableRow.findOne({ id_project: id_project, id_table: id_table, _id: id_row });
+    if (!row) return null;
+    return formatRow(row, table.schema || []);
+  }
+
+  async insertRow(id_project, id_table, rawData) {
+    return this.insertRowByBody(id_project, id_table, { data: rawData });
+  }
+
+  async insertRowByBody(id_project, id_table, body) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    if (body.data === undefined || body.data === null) {
+      throw new Error('data is required');
+    }
+    const data = dataTableUtiles.validateRowData(body.data, table.schema || []);
+    const rowDoc = buildInsertRowDoc(id_project, id_table, data, body.id_row);
+    const rowSize = computeInsertDelta(rowDoc);
+    assertTableSizeLimit(table, rowSize);
+
+    const row = await TableRow.create({
+      _id: rowDoc._id,
+      id_project: id_project,
+      id_table: id_table,
+      data: data,
+    });
+    await incrementTableStats(id_project, id_table, 1, rowSize);
+    return formatRow(row, table.schema || []);
+  }
+
+  async updateRow(id_project, id_table, id_row, rawData) {
+    return this.updateRowByBody(id_project, id_table, { id_row: id_row, data: rawData });
+  }
+
+  async updateRowByBody(id_project, id_table, body) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const schema = table.schema || [];
+    const update = dataTableUtiles.buildUpdateSet(body.data, schema);
+    const rowFilter = dataTableUtiles.buildRowQueryFilter(body, schema);
+    const filter = { id_project: id_project, id_table: id_table };
+    Object.assign(filter, rowFilter);
+
+    if (hasConditions(body)) {
+      const matches = await TableRow.find(filter).sort({ createdAt: 1 });
+      if (matches.length === 0) return undefined;
+      const sizeDelta = computeUpdateDelta(matches, update);
+      assertTableSizeLimit(table, sizeDelta);
+      await TableRow.updateMany(filter, update);
+      await incrementTableStats(id_project, id_table, 0, sizeDelta);
+      const updatedRows = await TableRow.find(filter).sort({ createdAt: 1 });
+      return updatedRows.map(function (r) { return formatRow(r, schema); });
+    }
+
+    const existing = await TableRow.findOne(filter);
+    if (!existing) return undefined;
+    const singleDelta = computeUpdateDelta([existing], update);
+    assertTableSizeLimit(table, singleDelta);
+    const row = await TableRow.findOneAndUpdate(filter, update, { new: true });
+    await incrementTableStats(id_project, id_table, 0, singleDelta);
+    return formatRow(row, schema);
+  }
+
+  async upsertRowByBody(id_project, id_table, body) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const schema = table.schema || [];
+    if (!body.data || typeof body.data !== 'object' || Array.isArray(body.data)) {
+      throw new Error('data is required');
+    }
+
+    const update = dataTableUtiles.buildUpdateSet(body.data, schema);
+    const rowData = dataTableUtiles.validateRowData(body.data, schema);
+
+    if (body.id_row) {
+      const filterById = { id_project: id_project, id_table: id_table, _id: body.id_row };
+      const existing = await TableRow.findOne(filterById);
+      if (existing) {
+        const updateDelta = computeUpdateDelta([existing], update);
+        assertTableSizeLimit(table, updateDelta);
+        const updated = await TableRow.findOneAndUpdate(filterById, update, { new: true });
+        await incrementTableStats(id_project, id_table, 0, updateDelta);
+        return formatRow(updated, schema);
+      }
+      const insertDoc = buildInsertRowDoc(id_project, id_table, rowData, body.id_row);
+      const insertSize = computeInsertDelta(insertDoc);
+      assertTableSizeLimit(table, insertSize);
+      try {
+        const created = await TableRow.create({
+          _id: body.id_row,
+          id_project: id_project,
+          id_table: id_table,
+          data: rowData,
+        });
+        await incrementTableStats(id_project, id_table, 1, insertSize);
+        return formatRow(created, schema);
+      } catch (createErr) {
+        if (createErr.code === 11000) {
+          const racedExisting = await TableRow.findOne(filterById);
+          if (racedExisting) {
+            const racedDelta = computeUpdateDelta([racedExisting], update);
+            assertTableSizeLimit(table, racedDelta);
+            const raced = await TableRow.findOneAndUpdate(filterById, update, { new: true });
+            await incrementTableStats(id_project, id_table, 0, racedDelta);
+            if (raced) return formatRow(raced, schema);
+          }
+        }
+        throw createErr;
+      }
+    }
+
+    const rowFilter = dataTableUtiles.buildRowQueryFilter(body, schema);
+    const filter = { id_project: id_project, id_table: id_table };
+    Object.assign(filter, rowFilter);
+    const matches = await TableRow.find(filter).sort({ createdAt: 1 });
+
+    if (matches.length === 0) {
+      const newDoc = buildInsertRowDoc(id_project, id_table, rowData);
+      const newSize = computeInsertDelta(newDoc);
+      assertTableSizeLimit(table, newSize);
+      const inserted = await TableRow.create({ id_project: id_project, id_table: id_table, data: rowData });
+      await incrementTableStats(id_project, id_table, 1, newSize);
+      return formatRow(inserted, schema);
+    }
+
+    if (matches.length > 1) {
+      throw new Error('Multiple rows match the conditions');
+    }
+
+    const upsertDelta = computeUpdateDelta(matches, update);
+    assertTableSizeLimit(table, upsertDelta);
+    const updated = await TableRow.findOneAndUpdate(filter, update, { new: true });
+    await incrementTableStats(id_project, id_table, 0, upsertDelta);
+    return formatRow(updated, schema);
+  }
+
+  async deleteRow(id_project, id_table, id_row) {
+    return this.deleteRowByBody(id_project, id_table, { id_row: id_row });
+  }
+
+  async deleteRowByBody(id_project, id_table, body) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const rowFilter = dataTableUtiles.buildRowQueryFilter(body, table.schema || []);
+    const filter = { id_project: id_project, id_table: id_table };
+    Object.assign(filter, rowFilter);
+
+    const schema = table.schema || [];
+
+    if (hasConditions(body)) {
+      const rows = await TableRow.find(filter).sort({ createdAt: 1 });
+      if (rows.length === 0) return undefined;
+      const formatted = rows.map(function (r) { return formatRow(r, schema); });
+      const deleteDelta = computeDeleteDelta(rows);
+      await TableRow.deleteMany(filter);
+      await incrementTableStats(id_project, id_table, deleteDelta.rowsDelta, deleteDelta.sizeDelta);
+      return formatted;
+    }
+
+    const row = await TableRow.findOne(filter);
+    if (!row) return undefined;
+    const singleDeleteDelta = computeDeleteDelta([row]);
+    const deleted = await TableRow.findOneAndDelete(filter);
+    await incrementTableStats(id_project, id_table, singleDeleteDelta.rowsDelta, singleDeleteDelta.sizeDelta);
+    return formatRow(deleted, schema);
+  }
+
+  async searchRows(id_project, id_table, searchBody) {
+    const table = await Table.findOne({ id_project: id_project, _id: id_table });
+    if (!table) return null;
+
+    const schema = table.schema || [];
+    const filter = dataTableUtiles.buildSearchFilter(searchBody, schema);
+    const query = { id_project: id_project, id_table: id_table };
+    Object.assign(query, filter);
+    const rows = await TableRow.find(query);
+    return rows.map(function (r) { return formatRow(r, schema); });
+  }
+}
+
+module.exports = new DataTableService();

package/services/requestService.js

@@ -255,13 +255,15 @@ class RequestService {
       //winston.info("main_flow_cache_3 route");
 
       // Find request
-      let query = Request.findOne({ request_id, id_project });
-      if (cacheEnabler.request) {
-        query = query.cache(cacheUtil.defaultTTL, id_project + ":requests:request_id:" + request_id + ":simple");
-        winston.debug('request cache enabled');
-      }
-      
-      const request = await query.exec();
+      // Cache disabled: a stale cached document can have an outdated __v and cause
+      // Mongoose VersionError on save ("No matching document found for id ... version N").
+      // let query = Request.findOne({ request_id, id_project });
+      // if (cacheEnabler.request) {
+      //   query = query.cache(cacheUtil.defaultTTL, id_project + ":requests:request_id:" + request_id + ":simple");
+      //   winston.debug('request cache enabled');
+      // }
+      // const request = await query.exec();
+      const request = await Request.findOne({ request_id, id_project }).exec();
 
       if (!request) {
         throw new Error(`Request not found: ${request_id}`);
@@ -401,15 +403,18 @@ class RequestService {
       // winston.debug("request_id", request_id);
       // winston.debug("newstatus", newstatus);
 
-      let q = Request
-        .findOne({ request_id: request_id, id_project: id_project });
-
-      if (cacheEnabler.request) {
-        q.cache(cacheUtil.defaultTTL, id_project + ":requests:request_id:" + request_id + ":simple")      //request_cache
-        winston.debug('request cache enabled');
-      }
-
-      return q.exec(function (err, request) {
+      // Cache disabled: a stale cached document can have an outdated __v and cause
+      // Mongoose VersionError on save ("No matching document found for id ... version N").
+      // let q = Request
+      //   .findOne({ request_id: request_id, id_project: id_project });
+      // if (cacheEnabler.request) {
+      //   q.cache(cacheUtil.defaultTTL, id_project + ":requests:request_id:" + request_id + ":simple");
+      //   winston.debug('request cache enabled');
+      // }
+      // return q.exec(function (err, request) {
+      return Request
+        .findOne({ request_id: request_id, id_project: id_project })
+        .exec(function (err, request) {
 
         if (err) {
           winston.error(err);