@airoom/nextmin-node 2.0.2 → 2.0.3

package/README.md

@@ -160,6 +160,150 @@ events.on(getModelEvent('Post', 'delete', 'before'), ({ id }) => {
 
 See full examples in documentation and the `examples/node` app inside this monorepo.
 
+## Cloudflare AI Search Integration (RAG Chatbot)
+
+NextMin includes a pluggable, cloud-native AI chatbot assistant widget powered entirely by **Cloudflare AI Search** (a fully managed cloud-based Retrieval-Augmented Generation platform). It supports crawling/scraping website pages, indexing database collections, and generating context-aware chat completions natively in the cloud.
+
+### 1. Environment Configuration
+
+Add the following environment variables to your `.env` file to enable Cloudflare AI Search:
+
+```env
+CLOUDFLARE_ACCOUNT_ID=your_cloudflare_account_id
+CLOUDFLARE_API_TOKEN=your_cloudflare_api_token
+CLOUDFLARE_AI_SEARCH_INDEX=your_ai_search_instance_name
+```
+
+### 2. Chat Widget Backend Setup
+
+Pass the `chatWidget` option when creating the NextMin router:
+
+```ts
+import { createNextMinRouter } from '@airoom/nextmin-node';
+
+const router = createNextMinRouter({
+  dbAdapter,
+  server, // Required for socket.io realtime events
+  chatWidget: {
+    enabled: true,
+    siteUrl: 'http://localhost:3000',                                  // Root URL to crawl & scrape
+    scrapePaths: ['/'],                                                // Entry points for crawler
+    frontendBuildPath: './.next',                                      // Local build directory (optional)
+    autoCrawlIntervalMs: 3600000,                                      // Auto-crawl interval (1 hour)
+    concurrency: 5                                                     // Concurrent crawler thread count (default: 5)
+  }
+});
+```
+
+*   **Zero Local server CPU/GPU footprint**: No GGUF or local model files are loaded in memory.
+*   **Pluggable Database Modularity**: No database tables or vector schemas (like `nextminembedding`) are registered when using Cloudflare AI Search, keeping your database 100% clean.
+*   **Scraped Staged Pages**: Scraped files are saved as `.txt` files in `.nextmin/scraped-pages/` on the local disk.
+*   **Cost-Efficient Incremental Crawls**: Hashes are tracked in `.nextmin/indexing-manifest.json` to only upload modified pages and delete removed ones.
+
+---
+
+### 3. Reindexing
+
+#### Programmatically in Code
+
+```ts
+import { reindex } from '@airoom/nextmin-node';
+
+await reindex({
+  chatWidget: {
+    enabled: true,
+    siteUrl: 'http://localhost:3000',
+    scrapePaths: ['/'],
+    concurrency: 5 // concurrent scraping threads (default: 5)
+  }
+});
+```
+
+#### Manual Reindexing via CLI
+
+To manually trigger a crawl and sync with Cloudflare AI Search, invoke the CLI directly:
+
+```bash
+# Using npx (globally available)
+npx nextmin reindex
+
+or
+
+npx nextmin reindex --force
+
+# Set custom concurrency threads
+npx nextmin reindex --threads 10
+
+# Or invoking the compiled JS file directly
+node ./node_modules/@airoom/nextmin-node/dist/cli.js reindex
+node ./node_modules/@airoom/nextmin-node/dist/cli.js reindex --force --threads 10
+```
+
+> **Note:** The CLI automatically loads your server's configuration from `.nextmin/config.json` (saved transparently on your server boot) so you do not need to duplicate setup params or provide database credentials. It also traverses directories upwards to load `.env.local` or `.env` files automatically.
+
+#### Environment Variables for Reindexing
+
+The `reindex` command reads the following environment variables:
+
+| Variable | Description | Default / Fallback |
+|---|---|---|
+| `NEXTMIN_CHAT_ENABLED` | Set to `false` to disable the indexer process completely. | `true` |
+| `CLOUDFLARE_ACCOUNT_ID` | Your Cloudflare Account ID (Required). | - |
+| `CLOUDFLARE_API_TOKEN` | Cloudflare API Token with vector/AI permissions (Required). | - |
+| `CLOUDFLARE_AI_SEARCH_INDEX` | Cloudflare AI Search index name (Required, falls back to `CLOUDFLARE_VECTORIZE_INDEX`). | - |
+| `NEXTMIN_CHAT_SITE_URL` | The full base website URL to crawl. | `process.env.FRONTEND_URL` \|\| `http://localhost:${PORT \|\| 3000}` |
+| `NEXTMIN_CHAT_SCRAPE_PATHS` | Comma-separated paths to scrape from the site URL. | `/` |
+| `NEXTMIN_CHAT_CONCURRENCY` | Number of concurrent scraping threads to run. | `5` |
+| `NEXTMIN_CHAT_FRONTEND_PATH` | Path to your frontend Next.js build directory. | `./.next` |
+| `NEXTMIN_CHAT_MODEL_PATH` | Path to a local GGUF model (if using local chat). | - |
+
+Alternatively, you can pass these directly as CLI flags (e.g. `--cf-account-id <id>`, `--cf-token <token>`, `--cf-index <index>`, `--site-url <url>`, `--threads <count>`).
+
+
+
+## Registration Whitelisting and Role Onboarding
+
+You can secure the user registration flow by configuring `publicRegisterFields` and `roleDefaultStatuses` in your `Users.json` schema:
+
+- **`publicRegisterFields`**: Array of fields allowed during public signup (e.g. `["username", "email", "firstName", "lastName", "password", "role"]`). If a field outside this list (like `status`) is submitted, a `400 Bad Request` is returned. If omitted, all fields are permitted for backward compatibility.
+- **`roleDefaultStatuses`**: Map of initial user statuses based on their selected role (e.g., `"buyer": "active"` to allow buyers immediately, and `"developer": "pending"` or `"agent": "pending"` to require administrator verification).
+- **Admin role protection**: Public endpoints block attempts to register under the `admin` or `superadmin` role (returns `403 Forbidden` unless requested by an authenticated administrator).
+
+## Schema Features: Dynamic `refPath` & `readOnly` Attributes
+
+### Dynamic References (`refPath`)
+To dynamically populate a reference based on another field's value in the same document (e.g. `subjectId` pointing to `Doctors` or `Hospitals` based on `subjectType`), specify `refPath` on the attribute definition:
+
+```json
+"subjectType": {
+  "type": "string",
+  "required": true,
+  "enum": ["Doctors", "Hospitals"],
+  "readOnly": true
+},
+"subjectId": {
+  "type": "ObjectId",
+  "refPath": "subjectType",
+  "show": "fullName",
+  "required": true,
+  "readOnly": true
+}
+```
+
+- **Mutually Exclusive with `ref`**: If `refPath` is defined, `refPath` takes precedence to resolve the reference target model dynamically per document. If `refPath` is absent, standard static `ref` is used as before.
+- **Strict Enum Constraint**: The attribute referenced by `refPath` (e.g. `subjectType`) MUST exist in the same schema and define an `enum` array of valid model names (e.g. `enum: ["Doctors", "Hospitals"]`).
+- **Developer Warnings**: During schema initialization, `nextmin-node` automatically validates `refPath` configurations and logs warnings in the terminal if:
+  1. The target field specified in `refPath` does not exist in the same schema.
+  2. The target field is not a string attribute with an `enum` array of target model names.
+  3. An `enum` string value does not correspond to any registered schema model.
+  4. Both `ref` and `refPath` are declared on the same attribute.
+- **Runtime Safety**: If a document has an invalid or missing `refPath` value or if the model is unpopulated, `nextmin-node` keeps the raw ID gracefully without throwing uncaught runtime exceptions.
+
+### Attribute Immutability (`readOnly`)
+Set `"readOnly": true` or `"editable": false` on an attribute definition. When present:
+- **API Level**: Update endpoints (`PUT /:model/:id`) strip read-only attributes from the payload automatically.
+- **Admin UI**: `@airoom/nextmin-react` renders disabled inputs in form views.
+
 ## Headers and auth
 
 - All requests must include the API key header: `x-api-key: <YOUR_API_KEY>`
@@ -182,6 +326,13 @@ Database
 - `MONGO_URL`: e.g. mongodb://localhost:27017
 - `MONGO_DB`: database name
 
+Cloudflare Vector Search (optional)
+
+- `CLOUDFLARE_ACCOUNT_ID`: Your Cloudflare Account ID
+- `CLOUDFLARE_API_TOKEN`: Cloudflare API Token with Vectorize & Workers AI permissions
+- `CLOUDFLARE_VECTORIZE_INDEX`: Cloudflare Vectorize index name
+- `CLOUDFLARE_EMBEDDING_MODEL`: Cloudflare Workers AI model name (e.g. `@cf/baai/bge-small-en-v1.5`)
+
 File storage (optional)
 
 - `S3_BUCKET`, `S3_REGION`

package/dist/api/apiRouter.d.ts

@@ -2,6 +2,21 @@ import express from 'express';
 import type { Server as HttpServer } from 'http';
 import { DatabaseAdapter } from '../database/DatabaseAdapter';
 import type { FileStorageAdapter } from '../files/FileStorageAdapter';
+export interface ChatWidgetConfig {
+    enabled: boolean;
+    whitelabelSchemas?: string[];
+    sourceType?: 'file' | 'db' | 'both';
+    siteUrl?: string;
+    scrapePaths?: string[];
+    autoCrawlIntervalMs?: number;
+    concurrency?: number;
+    sitemapUrl?: string;
+    systemPrompt?: string;
+    localModelPath?: string;
+    threads?: number;
+    enableOpenai?: boolean;
+    maxMessagesPerThread?: number;
+}
 export interface APIRouterOptions {
     dbAdapter: DatabaseAdapter;
     server?: HttpServer;
@@ -10,6 +25,7 @@ export interface APIRouterOptions {
         originalBase?: string;
         proxyBase?: string;
     };
+    chatWidget?: ChatWidgetConfig;
 }
 export declare class APIRouter {
     private router;
@@ -31,6 +47,10 @@ export declare class APIRouter {
     private realtimeService?;
     private fileRoutesMounted;
     private aggregateService?;
+    private chatWidgetConfig?;
+    private autoCrawlTimer?;
+    private isAutoCrawling;
+    private startAutoCrawlScheduler;
     constructor(options: APIRouterOptions);
     getRouter(): express.Router;
     private wireSchemaHotReload;

package/dist/api/apiRouter.js

@@ -1,4 +1,37 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -7,6 +40,8 @@ exports.APIRouter = void 0;
 const express_1 = __importDefault(require("express"));
 const BaseModel_1 = require("../models/BaseModel");
 const Logger_1 = __importDefault(require("../utils/Logger"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const jsonwebtoken_1 = __importDefault(require("jsonwebtoken"));
 const SchemaLoader_1 = require("../utils/SchemaLoader");
 const DefaultDataInitializer_1 = require("../utils/DefaultDataInitializer");
@@ -19,11 +54,48 @@ const mountCrudRoutes_1 = require("./router/mountCrudRoutes");
 const mountFindRoutes_1 = require("./router/mountFindRoutes");
 const mountSearchRoutes_1 = require("./router/mountSearchRoutes");
 const mountBatchRoutes_1 = require("./router/mountBatchRoutes");
+const setupChatWidgetRoutes_1 = require("./router/setupChatWidgetRoutes");
 class APIRouter {
     get isDevelopment() {
         return (process.env.APP_MODE?.toLowerCase() !== 'production' &&
             process.env.NODE_ENV?.toLowerCase() !== 'production');
     }
+    startAutoCrawlScheduler() {
+        const config = this.chatWidgetConfig;
+        if (!config ||
+            !config.enabled ||
+            process.env.DISABLE_AUTO_CRAWL === 'true' ||
+            process.env.NEXTMIN_DISABLE_AUTO_CRAWL === 'true' ||
+            !config.autoCrawlIntervalMs ||
+            config.autoCrawlIntervalMs <= 0) {
+            Logger_1.default.info('APIRouter', 'Auto-crawl scheduler is disabled (stop switch active or interval <= 0).');
+            return;
+        }
+        Logger_1.default.info('APIRouter', `Auto-crawl scheduler initialized. Interval: ${config.autoCrawlIntervalMs}ms (${(config.autoCrawlIntervalMs / 1000 / 60).toFixed(2)} minutes)`);
+        const runCrawl = async () => {
+            if (this.isAutoCrawling) {
+                Logger_1.default.warn('APIRouter', 'Previous auto-crawl run is still in progress. Skipping this interval.');
+                return;
+            }
+            this.isAutoCrawling = true;
+            Logger_1.default.info('APIRouter', 'Starting auto-crawl and reindexing...');
+            try {
+                const { reindex } = await Promise.resolve().then(() => __importStar(require('../services/IndexingService')));
+                await reindex({ chatWidget: config });
+                Logger_1.default.info('APIRouter', 'Auto-crawl and reindexing completed successfully.');
+            }
+            catch (err) {
+                Logger_1.default.error('APIRouter', `Auto-crawl and reindexing failed: ${err?.message || err}`);
+            }
+            finally {
+                this.isAutoCrawling = false;
+            }
+        };
+        // Run immediately on startup (2s delay to allow server to finish booting) - Disabled as requested to avoid startup overhead
+        // setTimeout(() => { runCrawl(); }, 2000);
+        // Then run on the recurring interval
+        this.autoCrawlTimer = setInterval(runCrawl, config.autoCrawlIntervalMs);
+    }
     constructor(options) {
         this.models = {};
         this.findRoutesMounted = false;
@@ -34,6 +106,7 @@ class APIRouter {
         this.registeredModels = new Set();
         this.liveSchemas = {};
         this.fileRoutesMounted = false;
+        this.isAutoCrawling = false;
         // ---------- Live lookups ----------
         this.getSchema = (name) => {
             const s = this.liveSchemas[name];
@@ -78,10 +151,16 @@ class APIRouter {
             const fromString = async (s) => {
                 if (!s)
                     return null;
-                if (isHex24(s) && rolesModel) {
-                    const docs = await rolesModel.read({ id: s }, 1, 0, true);
-                    const n = docs?.[0]?.name;
-                    return typeof n === 'string' ? n : null;
+                if (rolesModel) {
+                    const hexMatch = isHex24(s);
+                    const numericMatch = !isNaN(Number(s)) && s.trim() !== '';
+                    if (hexMatch || numericMatch) {
+                        const queryId = numericMatch ? Number(s) : s;
+                        const docs = await rolesModel.read({ id: queryId }, 1, 0, true);
+                        const n = docs?.[0]?.name;
+                        if (typeof n === 'string')
+                            return n;
+                    }
                 }
                 return s;
             };
@@ -191,6 +270,36 @@ class APIRouter {
         };
         this.router = express_1.default.Router();
         this.fileStorage = options.fileStorageAdapter;
+        this.chatWidgetConfig = options.chatWidget;
+        if (options.chatWidget) {
+            try {
+                const dir = path_1.default.join(process.cwd(), '.nextmin');
+                if (!fs_1.default.existsSync(dir)) {
+                    fs_1.default.mkdirSync(dir, { recursive: true });
+                }
+                const configPath = path_1.default.join(dir, 'config.json');
+                let existingConfig = {};
+                if (fs_1.default.existsSync(configPath)) {
+                    try {
+                        existingConfig = JSON.parse(fs_1.default.readFileSync(configPath, 'utf-8'));
+                    }
+                    catch { }
+                }
+                const mergedConfig = {
+                    ...options.chatWidget,
+                    enableOpenai: options.chatWidget.enableOpenai !== false,
+                };
+                if (existingConfig.openai) {
+                    mergedConfig.openai = existingConfig.openai;
+                }
+                fs_1.default.writeFileSync(configPath, JSON.stringify(mergedConfig, null, 2), 'utf-8');
+                Logger_1.default.info('APIRouter', `Saved merged chat widget config to .nextmin/config.json`);
+                this.chatWidgetConfig = mergedConfig;
+            }
+            catch (err) {
+                Logger_1.default.error('APIRouter', 'Failed to save chat widget config to disk', err);
+            }
+        }
         this.dbAdapter = options.dbAdapter;
         this.jwtSecret = process.env.JWT_SECRET || 'default_jwt_secret';
         if (options.server) {
@@ -205,7 +314,7 @@ class APIRouter {
         }
         this.schemaLoader =
             SchemaLoader_1.SchemaLoader.getInstance?.() ?? new SchemaLoader_1.SchemaLoader();
-        const initialSchemas = this.schemaLoader.getSchemas();
+        const initialSchemas = { ...this.schemaLoader.getSchemas() };
         this.setLiveSchemas(initialSchemas);
         const finishBoot = async () => {
             if (typeof this.dbAdapter.registerSchemas === 'function') {
@@ -233,6 +342,7 @@ class APIRouter {
             this.setupNotFoundMiddleware();
             this.aggregateService = new AggregateService_1.AggregateService(this.createCtx());
             this.wireSchemaHotReload();
+            this.startAutoCrawlScheduler();
         };
         if (typeof this.dbAdapter.registerSchemas === 'function') {
             const res = this.dbAdapter.registerSchemas(initialSchemas);
@@ -257,8 +367,9 @@ class APIRouter {
         const anyLoader = this.schemaLoader;
         if (typeof anyLoader.on !== 'function')
             return;
-        anyLoader.on('schemasChanged', async (newSchemas) => {
+        anyLoader.on('schemasChanged', async (newSchemasMap) => {
             try {
+                const newSchemas = { ...newSchemasMap };
                 const removed = this.diffRemovedModels(this.liveSchemas, newSchemas);
                 if (removed.length &&
                     typeof this.dbAdapter.unregisterSchemas === 'function') {
@@ -369,6 +480,8 @@ class APIRouter {
             (0, setupAuthRoutes_1.setupAuthRoutes)(this.createCtx());
             this.authRoutesInitialized = true;
         }
+        // Mount Chat Widget Routes
+        (0, setupChatWidgetRoutes_1.setupChatWidgetRoutes)(this.createCtx(), this.chatWidgetConfig);
         for (const s of Object.values(schemas)) {
             const name = s.modelName.toLowerCase();
             if (this.registeredModels.has(name))
@@ -503,10 +616,12 @@ class APIRouter {
         const conflictingFields = [];
         for (const field of uniqueFields) {
             const val = data[field];
+            if (val === undefined)
+                continue;
             const attr = schema.attributes?.[field];
             const isRequired = Array.isArray(attr) ? attr[0]?.required : attr?.required;
             // Sparse uniqueness: skip checking if value is empty/null AND the field is not required
-            if (val === undefined || val === null || (typeof val === 'string' && val.trim() === '')) {
+            if (val === null || (typeof val === 'string' && val.trim() === '')) {
                 if (!isRequired)
                     continue;
             }

package/dist/api/router/mountCrudRoutes.js

@@ -36,6 +36,35 @@ async function injectActionRelatedData(ctx, schema, rows) {
     }
 }
 const utils_1 = require("./utils");
+/**
+ * For User-extending schemas: strip inherited fields from a response doc
+ * when no base user is linked (baseId is null/empty).
+ * This keeps API responses clean — no null Username, First Name, etc.
+ */
+function stripInheritedIfNoBase(doc, schema) {
+    if (!doc || !schema?.extends)
+        return doc;
+    const isUserExt = String(schema.extends).toLowerCase() === 'users';
+    if (!isUserExt)
+        return doc;
+    const hasBase = !!(doc.baseId || doc.exId);
+    if (hasBase)
+        return doc;
+    const attrs = schema.attributes || {};
+    const result = { ...doc };
+    for (const [key, attr] of Object.entries(attrs)) {
+        const head = Array.isArray(attr) ? attr[0] : attr;
+        if (head?.inherited) {
+            delete result[key];
+        }
+    }
+    return result;
+}
+function stripInheritedIfNoBaseMany(docs, schema) {
+    if (!schema?.extends)
+        return docs;
+    return docs.map(d => stripInheritedIfNoBase(d, schema));
+}
 function mountCrudRoutes(ctx, modelNameLC) {
     const { router } = ctx;
     const basePath = `/${modelNameLC}`;
@@ -93,7 +122,14 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 return res.status(403).json({ error: true, message: 'forbidden' });
             }
             let payload = (0, authorize_1.mergeCreateDefaults)(req.body, cdec.createDefaults);
-            (0, authorize_1.enforceRestrictions)(payload, cdec.restrictions, pctx);
+            // Apply schema attribute defaults if missing in the request body
+            // For extended schemas, skip defaults for inherited attributes (they belong to the base record)
+            for (const [key, attr] of Object.entries(schema.attributes || {})) {
+                if (!Array.isArray(attr) && attr?.default !== undefined && payload[key] === undefined && !attr?.inherited) {
+                    payload[key] = attr.default;
+                }
+            }
+            await (0, authorize_1.enforceRestrictions)(payload, cdec.restrictions, pctx, ctx);
             if (!cdec.exposePrivate)
                 payload = (0, authorize_1.stripWriteDeny)(payload, cdec.writeDeny);
             payload = (0, fieldCodecs_1.coerceForStorage)(schema, payload);
@@ -126,6 +162,9 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 const basePayload = {};
                 const childPayload = {};
                 for (const [k, v] of Object.entries(payload)) {
+                    if (k === 'id' || k === '_id' || k === 'exId' || k === 'createdAt' || k === 'updatedAt') {
+                        continue;
+                    }
                     if (k === 'baseId') {
                         childPayload[k] = v;
                         continue;
@@ -164,7 +203,13 @@ function mountCrudRoutes(ctx, modelNameLC) {
                     // Only attempt creation if at least one base field is provided or if it's not a UserExt?
                     // Actually, for UserExt, if baseId is missing and they didn't fill user fields (which are hidden),
                     // we should skip base creation and just create the child.
-                    if (baseMissing.length === 0 && Object.keys(basePayload).length > 0) {
+                    // For User extensions: only create a base user if meaningful user fields
+                    // are explicitly provided (not just auto-injected defaults like status, type, role)
+                    const AUTO_DEFAULT_KEYS = new Set(['status', 'type', 'role', 'createdBy']);
+                    const hasMeaningfulBaseData = isUserExt
+                        ? Object.keys(basePayload).some(k => !AUTO_DEFAULT_KEYS.has(k))
+                        : Object.keys(basePayload).length > 0;
+                    if (baseMissing.length === 0 && hasMeaningfulBaseData) {
                         if (baseLC === 'users' && basePayload.password && ctx.jwtSecret) {
                             const salt = await bcrypt_1.default.genSalt(10);
                             basePayload.password = await bcrypt_1.default.hash(String(basePayload.password) + ctx.jwtSecret, salt);
@@ -231,7 +276,7 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 return res.status(201).json({
                     success: true,
                     message: `${schema.modelName} has been created successfully.`,
-                    data: masked,
+                    data: stripInheritedIfNoBase(masked, schema),
                 });
             }
             // non-extended
@@ -275,7 +320,7 @@ function mountCrudRoutes(ctx, modelNameLC) {
             }
             const model = ctx.getModel(modelNameLC);
             const baseKeys = new Set(Object.keys(schema.attributes || {}));
-            const { limit, page, skip, sort, projection } = (0, utils_1.parseQuery)(req, baseKeys);
+            const { limit, page, skip, sort, projection, _populate } = (0, utils_1.parseQuery)(req, baseKeys);
             const q = String(req.query.q ?? '').trim();
             const searchKey = String(req.query.searchKey ?? '').trim();
             const searchKeysCSV = String(req.query.searchKeys ?? '').trim();
@@ -330,13 +375,14 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 let dataRows = [];
                 if (requiresBaseProcessing) {
                     // If we need to filter/sort by base fields, read ALL child rows (already hydrated by adapter)
-                    dataRows = await model.read(childFilter, 0, 0, needPrivateForHydrate, { projection });
+                    dataRows = await model.read(childFilter, 0, 0, needPrivateForHydrate, { projection, _populate });
                 }
                 else {
                     // Normal paginated read (already hydrated by adapter)
                     dataRows = await model.read(childFilter, limit, skip, needPrivateForHydrate, {
                         sort: Object.keys(childSort).length ? childSort : undefined,
                         projection,
+                        _populate,
                     });
                 }
                 let merged = dataRows;
@@ -377,9 +423,6 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 }
                 // **dedupe**: one row per base entity
                 merged = (0, utils_1.dedupeBy)(merged, (r) => String(r.exId || r.id || r._id || ''));
-                if (currentUserId && isUsersModel) {
-                    merged = merged.filter((r) => String(r?.id) !== String(currentUserId));
-                }
                 // pagination + totals
                 let totalRows;
                 let paged;
@@ -400,7 +443,7 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 return res.status(200).json({
                     success: true,
                     message: `Data fetched for ${schema.modelName}`,
-                    data,
+                    data: stripInheritedIfNoBaseMany(data, schema),
                     pagination: { totalRows, page, limit },
                     sort: combinedSort,
                 });
@@ -410,7 +453,7 @@ function mountCrudRoutes(ctx, modelNameLC) {
             let rawRows = [];
             const exposePrivateForRead = !!rdec.exposePrivate;
             try {
-                rawRows = await model.read(finalFilter, limit + 1, skip, exposePrivateForRead, { sort, projection });
+                rawRows = await model.read(finalFilter, limit + 1, skip, exposePrivateForRead, { sort, projection, _populate });
             }
             catch {
                 rawRows = await model.read(finalFilter, limit + 1, skip, exposePrivateForRead);
@@ -438,11 +481,7 @@ function mountCrudRoutes(ctx, modelNameLC) {
                     });
                 }
             }
-            let rows = currentUserId && isUsersModel
-                ? rawRows
-                    .filter((r) => String(r?.id) !== String(currentUserId))
-                    .slice(0, limit)
-                : rawRows.slice(0, limit);
+            let rows = rawRows.slice(0, limit);
             const data = rdec.exposePrivate
                 ? (0, authorize_1.applyReadMaskMany)(rows, rdec.sensitiveMask)
                 : (0, authorize_1.applyReadMaskMany)(rows, rdec.readMask);
@@ -524,7 +563,7 @@ function mountCrudRoutes(ctx, modelNameLC) {
             return res.status(200).json({
                 success: true,
                 message: `${schema.modelName} found`,
-                data,
+                data: stripInheritedIfNoBase(data, schema),
             });
         }
         catch (error) {
@@ -556,9 +595,18 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 return res.status(403).json({ error: true, message: 'forbidden' });
             }
             let upd = { ...req.body };
-            (0, authorize_1.enforceRestrictions)(upd, udec.restrictions, pctx);
+            await (0, authorize_1.enforceRestrictions)(upd, udec.restrictions, pctx, ctx);
             if (!udec.exposePrivate)
                 upd = (0, authorize_1.stripWriteDeny)(upd, udec.writeDeny);
+            // Enforce schema readOnly / non-editable fields on update
+            if (schema?.attributes) {
+                for (const [attrName, attrDef] of Object.entries(schema.attributes)) {
+                    const a = Array.isArray(attrDef) ? attrDef[0] : attrDef;
+                    if (a?.readOnly === true || a?.editable === false) {
+                        delete upd[attrName];
+                    }
+                }
+            }
             upd = (0, fieldCodecs_1.coerceForStorage)(schema, upd);
             if (modelNameLC === 'users' &&
                 Object.prototype.hasOwnProperty.call(upd, 'password')) {
@@ -589,6 +637,9 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 const baseUpd = {};
                 const childUpd = {};
                 for (const [k, v] of Object.entries(upd)) {
+                    if (k === 'id' || k === '_id' || k === 'exId' || k === 'createdAt' || k === 'updatedAt') {
+                        continue;
+                    }
                     if (k === 'baseId') {
                         childUpd[k] = v;
                         continue;
@@ -680,7 +731,7 @@ function mountCrudRoutes(ctx, modelNameLC) {
                 const masked = udec.exposePrivate
                     ? (0, authorize_1.applyReadMaskOne)(responseDoc, udec.sensitiveMask)
                     : (0, authorize_1.applyReadMaskOne)(responseDoc, udec.readMask);
-                return res.json(masked);
+                return res.json(stripInheritedIfNoBase(masked, schema));
             }
             // non-extended
             const conflicts = await ctx.checkUniqueFields(schema, upd, req.params.id);