@mongoosejs/studio 0.5.4 → 0.5.6

package/backend/actions/Dashboard/createDashboard.js

@@ -2,6 +2,7 @@
 
 const Archetype = require('archetype');
 const authorize = require('../../authorize');
+const mongoose = require('mongoose');
 
 const CreateDashboardParams = new Archetype({
   title: {
@@ -12,18 +13,27 @@ const CreateDashboardParams = new Archetype({
     $type: 'string',
     $required: true
   },
+  initiatedById: {
+    $type: mongoose.Types.ObjectId,
+    $required: false
+  },
   roles: {
     $type: ['string']
   }
 }).compile('CreateDashboardParams');
 
 module.exports = ({ studioConnection }) => async function createDashboard(params) {
-  const { title, code, roles } = new CreateDashboardParams(params);
+  const { title, code, initiatedById, roles } = new CreateDashboardParams(params);
   const Dashboard = studioConnection.model('__Studio_Dashboard');
 
   await authorize('Dashboard.createDashboard', roles);
 
-  const dashboard = await Dashboard.create({ title, code });
+  const dashboard = await Dashboard.create({
+    title,
+    code,
+    createdById: initiatedById,
+    createdBy: params.initiatedBy
+  });
 
   return { dashboard };
 };

package/backend/actions/Dashboard/getDashboard.js

@@ -51,6 +51,7 @@ module.exports = ({ db, studioConnection, options }) => async function getDashbo
           return {};
         }
         return completeDashboardEvaluate(
+          Dashboard,
           DashboardResult,
           dashboardResult._id,
           null,
@@ -73,6 +74,7 @@ module.exports = ({ db, studioConnection, options }) => async function getDashbo
           return {};
         }
         return completeDashboardEvaluate(
+          Dashboard,
           DashboardResult,
           dashboardResult._id,
           result,
@@ -97,13 +99,15 @@ module.exports = ({ db, studioConnection, options }) => async function getDashbo
   }
 };
 
-async function completeDashboardEvaluate(DashboardResult, dashboardResultId, result, error, status) {
+async function completeDashboardEvaluate(Dashboard, DashboardResult, dashboardResultId, result, error, status) {
   const dashboardResult = await DashboardResult.findById(dashboardResultId).orFail();
-  dashboardResult.finishedEvaluatingAt = new Date();
+  const finishedEvaluatingAt = new Date();
+  dashboardResult.finishedEvaluatingAt = finishedEvaluatingAt;
   dashboardResult.result = result;
   dashboardResult.error = error;
   dashboardResult.status = status;
   await dashboardResult.save();
+  await Dashboard.updateOne({ _id: dashboardResult.dashboardId }, { $set: { lastEvaluatedAt: finishedEvaluatingAt } });
   return { dashboardResult };
 }
 

package/backend/actions/Dashboard/getDashboards.js

@@ -15,7 +15,9 @@ module.exports = ({ studioConnection }) => async function getDashboards(params)
 
   await authorize('Dashboard.getDashboards', roles);
 
-  const dashboards = await Dashboard.find();
+  const dashboards = await Dashboard
+    .find()
+    .sort({ isPinned: -1, createdAt: -1, _id: -1 });
 
   return { dashboards };
 };

package/backend/actions/Dashboard/updateDashboard.js

@@ -9,8 +9,7 @@ const UpdateDashboardParams = new Archetype({
     $required: true
   },
   code: {
-    $type: 'string',
-    $required: true
+    $type: 'string'
   },
   title: {
     $type: 'string'
@@ -18,19 +17,26 @@ const UpdateDashboardParams = new Archetype({
   description: {
     $type: 'string'
   },
+  isPinned: {
+    $type: 'boolean'
+  },
   roles: {
     $type: ['string']
   }
 }).compile('UpdateDashboardParams');
 
 module.exports = ({ studioConnection }) => async function updateDashboard(params) {
-  const { dashboardId, code, title, description, roles, evaluate } = new UpdateDashboardParams(params);
+  const { dashboardId, code, title, description, isPinned, roles, evaluate } = new UpdateDashboardParams(params);
 
   const Dashboard = studioConnection.models['__Studio_Dashboard'];
 
   await authorize('Dashboard.updateDashboard', roles);
 
-  const updateObj = { code };
+  const updateObj = {};
+
+  if (code != null) {
+    updateObj.code = code;
+  }
 
   if (title != null) {
     updateObj.title = title;
@@ -40,8 +46,15 @@ module.exports = ({ studioConnection }) => async function updateDashboard(params
     updateObj.description = description;
   }
 
-  const doc = await Dashboard.
-    findByIdAndUpdate(dashboardId, updateObj, { sanitizeFilter: true, returnDocument: 'after', overwriteImmutable: true });
+  if (isPinned != null) {
+    updateObj.isPinned = isPinned;
+  }
+
+  const doc = await Dashboard.findByIdAndUpdate(
+    dashboardId,
+    updateObj,
+    { sanitizeFilter: true, returnDocument: 'after' }
+  );
 
   return { doc };
 };

package/backend/actions/Model/analyzeSchema.js

@@ -0,0 +1,202 @@
+'use strict';
+
+const Archetype = require('archetype');
+const authorize = require('../../authorize');
+const { Buffer } = require('buffer');
+const validateDocumentWithTimeout = require('../../helpers/validateDocumentWithTimeout');
+
+const SAMPLE_SIZE = 1000;
+const MAX_SCHEMA_PATH_DEPTH = 4;
+
+const AnalyzeSchemaParams = new Archetype({
+  model: {
+    $type: 'string',
+    $required: true
+  },
+  roles: {
+    $type: ['string']
+  }
+}).compile('AnalyzeSchemaParams');
+
+module.exports = ({ db }) => async function analyzeSchema(params) {
+  const { model, roles } = new AnalyzeSchemaParams(params);
+
+  await authorize('Model.analyzeSchema', roles);
+
+  const Model = db.models[model];
+  if (Model == null) {
+    throw new Error(`Model ${model} not found`);
+  }
+
+  const documentCount = await Model.collection.countDocuments();
+  const rawDocs = documentCount > SAMPLE_SIZE ?
+    await Model.collection.aggregate([{ $sample: { size: SAMPLE_SIZE } }]).toArray() :
+    await Model.collection.find({}).toArray();
+
+  const paths = getSchemaPaths(Model.schema);
+  const pathTypeCounts = {};
+  const pathValueCounts = {};
+
+  for (const path of paths) {
+    pathTypeCounts[path.path] = {};
+    pathValueCounts[path.path] = 0;
+  }
+
+  const validationResults = await Promise.all(rawDocs.map(rawDoc => validateDocumentWithTimeout(Model.hydrate(rawDoc))));
+  const validDocumentCount = validationResults.filter(result => result.valid).length;
+  const firstValidDocument = validationResults.find(result => result.valid);
+  const firstInvalidDocument = validationResults.find(result => !result.valid);
+
+  for (const rawDoc of rawDocs) {
+    for (const path of paths) {
+      for (const value of getPathValues(rawDoc, path.path, path.arrayParents)) {
+        const types = path.arrayPath ? [getType(value)] : getTypes(value);
+        pathValueCounts[path.path] += types.length;
+        for (const type of types) {
+          pathTypeCounts[path.path][type] = (pathTypeCounts[path.path][type] || 0) + 1;
+        }
+      }
+    }
+  }
+
+  return {
+    analysis: {
+      documentCount,
+      sampleSize: rawDocs.length,
+      sampled: documentCount > SAMPLE_SIZE,
+      validDocumentCount,
+      invalidDocumentCount: rawDocs.length - validDocumentCount,
+      firstValidDocumentId: firstValidDocument?.documentId || null,
+      firstInvalidDocumentId: firstInvalidDocument?.documentId || null,
+      invalidDocuments: validationResults.filter(result => !result.valid).map(result => ({
+        documentId: result.documentId,
+        error: result.error,
+        errors: result.errors
+      })),
+      paths: paths.map(path => ({
+        path: path.path,
+        valueCount: pathValueCounts[path.path],
+        ...(path.required ? { required: true } : {}),
+        types: Object.keys(pathTypeCounts[path.path]).map(type => ({
+          type,
+          count: pathTypeCounts[path.path][type]
+        })).sort((type1, type2) => type2.count - type1.count)
+      }))
+    }
+  };
+};
+
+function getSchemaPaths(schema, prefix, parentArrayParents) {
+  prefix = prefix || '';
+  parentArrayParents = parentArrayParents || [];
+
+  const paths = [];
+  for (const path of Object.keys(schema.paths)) {
+    const fullPath = prefix ? `${prefix}.${path}` : path;
+    const pathDepth = fullPath.split('.').length;
+    if (pathDepth > MAX_SCHEMA_PATH_DEPTH) {
+      continue;
+    }
+    const schemaType = schema.paths[path];
+    const arrayParents = parentArrayParents.concat(getSchemaArrayParents(schemaType, fullPath));
+    paths.push({
+      path: fullPath,
+      arrayParents,
+      arrayPath: isSchemaArrayPath(schemaType),
+      required: schemaType.options?.required === true
+    });
+
+    if (schemaType.schema && pathDepth < MAX_SCHEMA_PATH_DEPTH) {
+      paths.push(...getSchemaPaths(schemaType.schema, fullPath, arrayParents));
+    }
+  }
+  return paths;
+}
+
+function getSchemaArrayParents(schemaType, fullPath) {
+  if (isSchemaArrayPath(schemaType)) {
+    return [fullPath];
+  }
+  const parent = schemaType.$parentSchemaDocArray;
+  if (parent?.path) {
+    return [parent.path];
+  }
+  return [];
+}
+
+function isSchemaArrayPath(schemaType) {
+  return schemaType.$isMongooseDocumentArray || schemaType.$isMongooseArray || schemaType.instance === 'Array';
+}
+
+function getPathValues(doc, path, arrayParents) {
+  return getPathValuesRecursive(doc, path.split('.'), '', new Set(arrayParents));
+}
+
+function getPathValuesRecursive(value, parts, currentPath, arrayParents) {
+  if (parts.length === 0) {
+    return [value];
+  }
+
+  if (Array.isArray(value)) {
+    return value.flatMap(element => getPathValuesRecursive(element, parts, currentPath, arrayParents));
+  }
+
+  if (value == null) {
+    return [undefined];
+  }
+
+  const nextPart = parts[0];
+  const nextPath = currentPath ? `${currentPath}.${nextPart}` : nextPart;
+  const nextValue = getPathPartValue(value, nextPart);
+  if (nextValue == null && arrayParents.has(nextPath)) {
+    return [];
+  }
+
+  return getPathValuesRecursive(nextValue, parts.slice(1), nextPath, arrayParents);
+}
+
+function getPathPartValue(value, pathPart) {
+  if (value instanceof Map) {
+    return value.get(pathPart);
+  }
+  return value[pathPart];
+}
+
+function getTypes(value) {
+  if (!Array.isArray(value)) {
+    return [getType(value)];
+  }
+  if (value.length === 0) {
+    return ['array'];
+  }
+  const types = new Set();
+  for (const element of value) {
+    types.add(getType(element));
+  }
+  return Array.from(types);
+}
+
+function getType(value) {
+  if (value === undefined) {
+    return 'undefined';
+  }
+  if (value === null) {
+    return 'null';
+  }
+  if (Array.isArray(value)) {
+    return 'array';
+  }
+  if (value instanceof Date) {
+    return 'date';
+  }
+  if (Buffer.isBuffer(value)) {
+    return 'buffer';
+  }
+  if (value?._bsontype === 'ObjectId') {
+    return 'objectId';
+  }
+  if (value?._bsontype === 'Decimal128') {
+    return 'decimal128';
+  }
+  return typeof value;
+}

package/backend/actions/Model/index.js

@@ -1,6 +1,7 @@
 'use strict';
 
 exports.addField = require('./addField');
+exports.analyzeSchema = require('./analyzeSchema');
 exports.createChatMessage = require('./createChatMessage');
 exports.createDocument = require('./createDocument');
 exports.deleteDocument = require('./deleteDocument');
@@ -21,3 +22,4 @@ exports.streamDocumentChanges = require('./streamDocumentChanges');
 exports.streamChatMessage = require('./streamChatMessage');
 exports.updateDocument = require('./updateDocument');
 exports.updateDocuments = require('./updateDocuments');
+exports.validateDocument = require('./validateDocument');

package/backend/actions/Model/validateDocument.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const Archetype = require('archetype');
+const authorize = require('../../authorize');
+const validateDocumentWithTimeout = require('../../helpers/validateDocumentWithTimeout');
+
+const ValidateDocumentParams = new Archetype({
+  model: {
+    $type: 'string',
+    $required: true
+  },
+  documentId: {
+    $type: 'string',
+    $required: true
+  },
+  roles: {
+    $type: ['string']
+  }
+}).compile('ValidateDocumentParams');
+
+module.exports = ({ db }) => async function validateDocument(params) {
+  const { model, documentId, roles } = new ValidateDocumentParams(params);
+
+  await authorize('Model.validateDocument', roles);
+
+  const Model = db.models[model];
+  if (Model == null) {
+    throw new Error(`Model ${model} not found`);
+  }
+
+  const doc = await Model.findById(documentId).setOptions({ sanitizeFilter: true }).orFail();
+  return {
+    result: await validateDocumentWithTimeout(doc)
+  };
+};

package/backend/authorize.js

@@ -14,7 +14,9 @@ const actionsToRequiredRoles = {
   'Dashboard.getDashboards': ['owner', 'admin', 'member', 'readonly', 'dashboards'],
   'Dashboard.updateDashboard': ['owner', 'admin', 'member'],
   'Model.createDocument': ['owner', 'admin', 'member'],
+  'Model.analyzeSchema': ['owner', 'admin', 'member', 'readonly'],
   'Model.updateDocument': ['owner', 'admin', 'member'],
+  'Model.validateDocument': ['owner', 'admin', 'member', 'readonly'],
   'Model.deleteDocument': ['owner', 'admin', 'member'],
   'Model.deleteDocuments': ['owner', 'admin', 'member'],
   'Model.dropCollection': ['owner', 'admin'],

package/backend/db/dashboardSchema.js

@@ -13,6 +13,37 @@ const dashboardSchema = new mongoose.Schema({
   },
   description: {
     type: String
+  },
+  isPinned: {
+    type: Boolean
+  },
+  createdById: {
+    type: mongoose.Schema.Types.ObjectId
+  },
+  createdBy: {
+    name: String,
+    email: String
+  },
+  lastEvaluatedAt: {
+    type: Date
+  }
+}, { timestamps: true });
+
+dashboardSchema.post(['find', 'findOne'], async function(docs) {
+  const dashboards = Array.isArray(docs) ? docs : [docs];
+  for (const dashboard of dashboards) {
+    if (dashboard != null && dashboard.createdAt == null && dashboard._id?.getTimestamp) {
+      const createdAt = dashboard._id.getTimestamp();
+      if (!createdAt.valueOf()) {
+        continue;
+      }
+      dashboard.set('createdAt', createdAt, { overwriteImmutable: true });
+      await this.model.db.model('__Studio_Dashboard').updateOne(
+        { _id: dashboard._id },
+        { createdAt },
+        { overwriteImmutable: true }
+      );
+    }
   }
 });
 

package/backend/helpers/validateDocumentWithTimeout.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const VALIDATION_TIMEOUT_MS = 1000;
+
+module.exports = async function validateDocumentWithTimeout(doc) {
+  const documentId = doc._id?.toString() || null;
+  try {
+    await Promise.race([
+      doc.validate(),
+      new Promise((resolve, reject) => setTimeout(() => reject(new Error('Validation timed out')), VALIDATION_TIMEOUT_MS))
+    ]);
+    return { valid: true, documentId, error: null, errors: null };
+  } catch (err) {
+    return {
+      valid: false,
+      documentId,
+      error: err?.message || String(err),
+      errors: formatValidationErrors(err)
+    };
+  }
+};
+
+function formatValidationErrors(err) {
+  if (err?.name !== 'ValidationError' || err.errors == null) {
+    return null;
+  }
+
+  const errors = {};
+  for (const [path, pathError] of Object.entries(err.errors)) {
+    errors[path] = {
+      message: pathError?.message || String(pathError),
+      name: pathError?.name || null,
+      kind: pathError?.kind || null,
+      path: pathError?.path || path,
+      value: pathError?.value
+    };
+  }
+  return errors;
+}

package/backend/netlify.js

@@ -63,6 +63,7 @@ module.exports = function netlify(conn, options) {
       params.roles = roles;
       params.userId = user._id;
       params.initiatedById = user._id;
+      params.initiatedBy = user;
     }
 
     if (typeof actionName !== 'string') {

package/backend/next.js

@@ -64,6 +64,7 @@ module.exports = function next(conn, options) {
       params.roles = roles;
       params.userId = user._id;
       params.initiatedById = user._id;
+      params.initiatedBy = user;
     }
 
     if (typeof actionName !== 'string') {

package/express.js

@@ -87,6 +87,7 @@ module.exports = async function mongooseStudioExpressApp(apiUrl, conn, options)
           req._internals.initiatedById = user._id;
           req._internals.roles = roles;
           req._internals.$workspaceId = workspace._id;
+          req._internals.initiatedBy = user;
 
           next();
         })