@objectstack/objectql 3.0.11 → 3.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@objectstack/objectql",
-  "version": "3.0.11",
+  "version": "3.1.0",
   "license": "Apache-2.0",
   "description": "Isomorphic ObjectQL Engine for ObjectStack",
   "main": "dist/index.js",
@@ -13,9 +13,9 @@
     }
   },
   "dependencies": {
-    "@objectstack/core": "3.0.11",
-    "@objectstack/spec": "3.0.11",
-    "@objectstack/types": "3.0.11"
+    "@objectstack/core": "3.1.0",
+    "@objectstack/spec": "3.1.0",
+    "@objectstack/types": "3.1.0"
   },
   "devDependencies": {
     "typescript": "^5.0.0",

package/src/engine.test.ts

@@ -210,4 +210,390 @@ describe('ObjectQL Engine', () => {
             expect(result).toHaveLength(1);
         });
     });
+
+    describe('Expand Related Records', () => {
+        beforeEach(async () => {
+            engine.registerDriver(mockDriver, true);
+            await engine.init();
+        });
+
+        it('should expand lookup fields by replacing IDs with full objects', async () => {
+            // Setup: task has a lookup field "assignee" → user object
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        assignee: { type: 'lookup', reference: 'user' },
+                        title: { type: 'text' },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {
+                        name: { type: 'text' },
+                    },
+                } as any;
+                return undefined;
+            });
+
+            // Primary find returns tasks with assignee IDs
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([
+                    { _id: 't1', title: 'Task 1', assignee: 'u1' },
+                    { _id: 't2', title: 'Task 2', assignee: 'u2' },
+                ])
+                // Second call (expand): returns user records
+                .mockResolvedValueOnce([
+                    { _id: 'u1', name: 'Alice' },
+                    { _id: 'u2', name: 'Bob' },
+                ]);
+
+            const result = await engine.find('task', { populate: ['assignee'] });
+
+            expect(result).toHaveLength(2);
+            expect(result[0].assignee).toEqual({ _id: 'u1', name: 'Alice' });
+            expect(result[1].assignee).toEqual({ _id: 'u2', name: 'Bob' });
+
+            // Verify the expand query used $in
+            expect(mockDriver.find).toHaveBeenCalledTimes(2);
+            expect(mockDriver.find).toHaveBeenLastCalledWith(
+                'user',
+                expect.objectContaining({
+                    object: 'user',
+                    where: { _id: { $in: ['u1', 'u2'] } },
+                }),
+            );
+        });
+
+        it('should expand master_detail fields', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'order_item') return {
+                    name: 'order_item',
+                    fields: {
+                        order: { type: 'master_detail', reference: 'order' },
+                    },
+                } as any;
+                if (name === 'order') return {
+                    name: 'order',
+                    fields: { total: { type: 'number' } },
+                } as any;
+                return undefined;
+            });
+
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([
+                    { _id: 'oi1', order: 'o1' },
+                ])
+                .mockResolvedValueOnce([
+                    { _id: 'o1', total: 100 },
+                ]);
+
+            const result = await engine.find('order_item', { populate: ['order'] });
+            expect(result[0].order).toEqual({ _id: 'o1', total: 100 });
+        });
+
+        it('should skip expand for fields without reference definition', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockReturnValue({
+                name: 'task',
+                fields: {
+                    title: { type: 'text' }, // Not a lookup
+                },
+            } as any);
+
+            vi.mocked(mockDriver.find).mockResolvedValueOnce([
+                { _id: 't1', title: 'Task 1' },
+            ]);
+
+            const result = await engine.find('task', { populate: ['title'] });
+            expect(result[0].title).toBe('Task 1'); // Unchanged
+            expect(mockDriver.find).toHaveBeenCalledTimes(1); // No expand query
+        });
+
+        it('should skip expand if schema is not registered', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockReturnValue(undefined);
+
+            vi.mocked(mockDriver.find).mockResolvedValueOnce([
+                { _id: 't1', assignee: 'u1' },
+            ]);
+
+            const result = await engine.find('task', { populate: ['assignee'] });
+            expect(result[0].assignee).toBe('u1'); // Unchanged — raw ID
+            expect(mockDriver.find).toHaveBeenCalledTimes(1);
+        });
+
+        it('should handle null values gracefully during expand', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        assignee: { type: 'lookup', reference: 'user' },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {},
+                } as any;
+                return undefined;
+            });
+
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([
+                    { _id: 't1', assignee: null },
+                    { _id: 't2', assignee: 'u1' },
+                ])
+                .mockResolvedValueOnce([
+                    { _id: 'u1', name: 'Alice' },
+                ]);
+
+            const result = await engine.find('task', { populate: ['assignee'] });
+            expect(result[0].assignee).toBeNull();
+            expect(result[1].assignee).toEqual({ _id: 'u1', name: 'Alice' });
+        });
+
+        it('should de-duplicate foreign key IDs in batch query', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        assignee: { type: 'lookup', reference: 'user' },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {},
+                } as any;
+                return undefined;
+            });
+
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([
+                    { _id: 't1', assignee: 'u1' },
+                    { _id: 't2', assignee: 'u1' }, // Same user
+                    { _id: 't3', assignee: 'u2' },
+                ])
+                .mockResolvedValueOnce([
+                    { _id: 'u1', name: 'Alice' },
+                    { _id: 'u2', name: 'Bob' },
+                ]);
+
+            const result = await engine.find('task', { populate: ['assignee'] });
+
+            // Verify only 2 unique IDs queried
+            expect(mockDriver.find).toHaveBeenLastCalledWith(
+                'user',
+                expect.objectContaining({
+                    where: { _id: { $in: ['u1', 'u2'] } },
+                }),
+            );
+            expect(result[0].assignee).toEqual({ _id: 'u1', name: 'Alice' });
+            expect(result[1].assignee).toEqual({ _id: 'u1', name: 'Alice' });
+        });
+
+        it('should keep raw ID when referenced record not found', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        assignee: { type: 'lookup', reference: 'user' },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {},
+                } as any;
+                return undefined;
+            });
+
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([
+                    { _id: 't1', assignee: 'u_deleted' },
+                ])
+                .mockResolvedValueOnce([]); // No records found
+
+            const result = await engine.find('task', { populate: ['assignee'] });
+            expect(result[0].assignee).toBe('u_deleted'); // Fallback to raw ID
+        });
+
+        it('should expand multiple fields in a single query', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        assignee: { type: 'lookup', reference: 'user' },
+                        project: { type: 'lookup', reference: 'project' },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {},
+                } as any;
+                if (name === 'project') return {
+                    name: 'project',
+                    fields: {},
+                } as any;
+                return undefined;
+            });
+
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([
+                    { _id: 't1', assignee: 'u1', project: 'p1' },
+                ])
+                .mockResolvedValueOnce([{ _id: 'u1', name: 'Alice' }])
+                .mockResolvedValueOnce([{ _id: 'p1', name: 'Project X' }]);
+
+            const result = await engine.find('task', { populate: ['assignee', 'project'] });
+
+            expect(result[0].assignee).toEqual({ _id: 'u1', name: 'Alice' });
+            expect(result[0].project).toEqual({ _id: 'p1', name: 'Project X' });
+            expect(mockDriver.find).toHaveBeenCalledTimes(3);
+        });
+
+        it('should work with findOne and expand', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        assignee: { type: 'lookup', reference: 'user' },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {},
+                } as any;
+                return undefined;
+            });
+
+            vi.mocked(mockDriver.findOne as any).mockResolvedValueOnce(
+                { _id: 't1', title: 'Task 1', assignee: 'u1' },
+            );
+            vi.mocked(mockDriver.find).mockResolvedValueOnce([
+                { _id: 'u1', name: 'Alice' },
+            ]);
+
+            const result = await engine.findOne('task', { populate: ['assignee'] });
+
+            expect(result.assignee).toEqual({ _id: 'u1', name: 'Alice' });
+        });
+
+        it('should handle already-expanded objects (skip re-expansion)', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        assignee: { type: 'lookup', reference: 'user' },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {},
+                } as any;
+                return undefined;
+            });
+
+            // Driver returns an already-expanded object
+            vi.mocked(mockDriver.find).mockResolvedValueOnce([
+                { _id: 't1', assignee: { _id: 'u1', name: 'Alice' } },
+            ]);
+
+            const result = await engine.find('task', { populate: ['assignee'] });
+
+            // No expand query should have been made — the value was already an object
+            expect(mockDriver.find).toHaveBeenCalledTimes(1);
+            expect(result[0].assignee).toEqual({ _id: 'u1', name: 'Alice' });
+        });
+
+        it('should gracefully handle expand errors and keep raw IDs', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        assignee: { type: 'lookup', reference: 'user' },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {},
+                } as any;
+                return undefined;
+            });
+
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([
+                    { _id: 't1', assignee: 'u1' },
+                ])
+                .mockRejectedValueOnce(new Error('Driver connection failed'));
+
+            const result = await engine.find('task', { populate: ['assignee'] });
+            expect(result[0].assignee).toBe('u1'); // Kept raw ID
+        });
+
+        it('should handle multi-value lookup fields (arrays)', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                if (name === 'task') return {
+                    name: 'task',
+                    fields: {
+                        watchers: { type: 'lookup', reference: 'user', multiple: true },
+                    },
+                } as any;
+                if (name === 'user') return {
+                    name: 'user',
+                    fields: {},
+                } as any;
+                return undefined;
+            });
+
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([
+                    { _id: 't1', watchers: ['u1', 'u2'] },
+                ])
+                .mockResolvedValueOnce([
+                    { _id: 'u1', name: 'Alice' },
+                    { _id: 'u2', name: 'Bob' },
+                ]);
+
+            const result = await engine.find('task', { populate: ['watchers'] });
+            expect(result[0].watchers).toEqual([
+                { _id: 'u1', name: 'Alice' },
+                { _id: 'u2', name: 'Bob' },
+            ]);
+        });
+
+        it('should expand only fields specified in the expand map (populate creates flat expand)', async () => {
+            // populate: ['project'] creates expand: { project: { object: 'project' } } (1 level only)
+            // Nested fields like project.org should NOT be expanded unless explicitly nested in the AST
+            vi.mocked(SchemaRegistry.getObject).mockImplementation((name) => {
+                const schemas: Record<string, any> = {
+                    task: { name: 'task', fields: { project: { type: 'lookup', reference: 'project' } } },
+                    project: { name: 'project', fields: { org: { type: 'lookup', reference: 'org' } } },
+                };
+                return schemas[name] as any;
+            });
+
+            vi.mocked(mockDriver.find)
+                .mockResolvedValueOnce([{ _id: 't1', project: 'p1' }])  // find task
+                .mockResolvedValueOnce([{ _id: 'p1', org: 'o1' }]);     // expand project (depth 0)
+                // org should NOT be expanded further — flat populate doesn't create nested expand
+
+            const result = await engine.find('task', { populate: ['project'] });
+
+            // Project expanded, but org inside project remains as raw ID
+            expect(result[0].project).toEqual({ _id: 'p1', org: 'o1' });
+            expect(mockDriver.find).toHaveBeenCalledTimes(2); // Only primary + 1 expand query
+        });
+
+        it('should return records unchanged when expand map is empty', async () => {
+            vi.mocked(SchemaRegistry.getObject).mockReturnValue({
+                name: 'task',
+                fields: {},
+            } as any);
+
+            vi.mocked(mockDriver.find).mockResolvedValueOnce([
+                { _id: 't1', title: 'Task 1' },
+            ]);
+
+            const result = await engine.find('task', {});
+            expect(result).toEqual([{ _id: 't1', title: 'Task 1' }]);
+            expect(mockDriver.find).toHaveBeenCalledTimes(1);
+        });
+    });
 });

package/src/engine.ts

@@ -617,6 +617,127 @@ export class ObjectQL implements IDataEngine {
     this.logger.info('ObjectQL engine destroyed');
   }
 
+  // ============================================
+  // Helper: Expand Related Records
+  // ============================================
+
+  /** Maximum depth for recursive expand to prevent infinite loops */
+  private static readonly MAX_EXPAND_DEPTH = 3;
+
+  /**
+   * Post-process expand: resolve lookup/master_detail fields by batch-loading related records.
+   * 
+   * This is a driver-agnostic implementation that uses secondary queries ($in batches)
+   * to load related records, then injects them into the result set.
+   * 
+   * @param objectName - The source object name
+   * @param records - The records returned by the driver
+   * @param expand - The expand map from QueryAST (field name → nested QueryAST)
+   * @param depth - Current recursion depth (0-based)
+   * @returns Records with expanded lookup fields (IDs replaced by full objects)
+   */
+  private async expandRelatedRecords(
+    objectName: string,
+    records: any[],
+    expand: Record<string, QueryAST>,
+    depth: number = 0,
+  ): Promise<any[]> {
+    if (!records || records.length === 0) return records;
+    if (depth >= ObjectQL.MAX_EXPAND_DEPTH) return records;
+
+    const objectSchema = SchemaRegistry.getObject(objectName);
+    // If no schema registered, skip expand — return raw data
+    if (!objectSchema || !objectSchema.fields) return records;
+
+    for (const [fieldName, nestedAST] of Object.entries(expand)) {
+      const fieldDef = objectSchema.fields[fieldName];
+
+      // Skip if field not found or not a relationship type
+      if (!fieldDef || !fieldDef.reference) continue;
+      if (fieldDef.type !== 'lookup' && fieldDef.type !== 'master_detail') continue;
+
+      const referenceObject = fieldDef.reference;
+
+      // Collect all foreign key IDs from records (handle both single and multiple values)
+      const allIds: any[] = [];
+      for (const record of records) {
+        const val = record[fieldName];
+        if (val == null) continue;
+        if (Array.isArray(val)) {
+          allIds.push(...val.filter((id: any) => id != null));
+        } else if (typeof val === 'object') {
+          // Already expanded — skip
+          continue;
+        } else {
+          allIds.push(val);
+        }
+      }
+
+      // De-duplicate IDs
+      const uniqueIds = [...new Set(allIds)];
+      if (uniqueIds.length === 0) continue;
+
+      // Batch-load related records using $in query
+      try {
+        const relatedQuery: QueryAST = {
+          object: referenceObject,
+          where: { _id: { $in: uniqueIds } },
+          ...(nestedAST.fields ? { fields: nestedAST.fields } : {}),
+          ...(nestedAST.orderBy ? { orderBy: nestedAST.orderBy } : {}),
+        };
+
+        const driver = this.getDriver(referenceObject);
+        const relatedRecords = await driver.find(referenceObject, relatedQuery) ?? [];
+
+        // Build a lookup map: id → record
+        const recordMap = new Map<string, any>();
+        for (const rec of relatedRecords) {
+          const id = rec._id ?? rec.id;
+          if (id != null) recordMap.set(String(id), rec);
+        }
+
+        // Recursively expand nested relations if present
+        if (nestedAST.expand && Object.keys(nestedAST.expand).length > 0) {
+          const expandedRelated = await this.expandRelatedRecords(
+            referenceObject,
+            relatedRecords,
+            nestedAST.expand,
+            depth + 1,
+          );
+          // Rebuild map with expanded records
+          recordMap.clear();
+          for (const rec of expandedRelated) {
+            const id = rec._id ?? rec.id;
+            if (id != null) recordMap.set(String(id), rec);
+          }
+        }
+
+        // Inject expanded records back into the original result set
+        for (const record of records) {
+          const val = record[fieldName];
+          if (val == null) continue;
+
+          if (Array.isArray(val)) {
+            record[fieldName] = val.map((id: any) => recordMap.get(String(id)) ?? id);
+          } else if (typeof val !== 'object') {
+            record[fieldName] = recordMap.get(String(val)) ?? val;
+          }
+          // If val is already an object, leave it as-is
+        }
+      } catch (e) {
+        // Graceful degradation: if expand fails, keep original IDs
+        this.logger.warn('Failed to expand relationship field; retaining foreign key IDs', {
+          object: objectName,
+          field: fieldName,
+          reference: referenceObject,
+          error: (e as Error).message,
+        });
+      }
+    }
+
+    return records;
+  }
+
   // ============================================
   // Helper: Query Conversion
   // ============================================
@@ -691,7 +812,12 @@ export class ObjectQL implements IDataEngine {
       await this.triggerHooks('beforeFind', hookContext);
 
       try {
-          const result = await driver.find(object, hookContext.input.ast as QueryAST, hookContext.input.options as any);
+          let result = await driver.find(object, hookContext.input.ast as QueryAST, hookContext.input.options as any);
+
+          // Post-process: expand related records if expand is requested
+          if (ast.expand && Object.keys(ast.expand).length > 0 && Array.isArray(result)) {
+            result = await this.expandRelatedRecords(object, result, ast.expand, 0);
+          }
           
           hookContext.event = 'afterFind';
           hookContext.result = result;
@@ -723,7 +849,15 @@ export class ObjectQL implements IDataEngine {
     };
 
     await this.executeWithMiddleware(opCtx, async () => {
-      return driver.findOne(objectName, opCtx.ast as QueryAST);
+      let result = await driver.findOne(objectName, opCtx.ast as QueryAST);
+
+      // Post-process: expand related records if expand is requested
+      if (ast.expand && Object.keys(ast.expand).length > 0 && result != null) {
+        const expanded = await this.expandRelatedRecords(objectName, [result], ast.expand, 0);
+        result = expanded[0];
+      }
+
+      return result;
     });
 
     return opCtx.result;

package/src/protocol.ts

@@ -327,6 +327,19 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
             options.populate = options.populate.split(',').map((s: string) => s.trim()).filter(Boolean);
         }
 
+        // Expand → Populate: normalize $expand/expand to populate array
+        // Supports OData $expand, REST expand, and JSON-RPC expand parameters
+        const expandValue = options.$expand ?? options.expand;
+        if (expandValue && !options.populate) {
+            if (typeof expandValue === 'string') {
+                options.populate = expandValue.split(',').map((s: string) => s.trim()).filter(Boolean);
+            } else if (Array.isArray(expandValue)) {
+                options.populate = expandValue;
+            }
+        }
+        delete options.$expand;
+        delete options.expand;
+
         // Boolean fields
         for (const key of ['distinct', 'count']) {
             if (options[key] === 'true') options[key] = true;
@@ -346,10 +359,26 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
         };
     }
 
-    async getData(request: { object: string, id: string }) {
-        const result = await this.engine.findOne(request.object, {
+    async getData(request: { object: string, id: string, expand?: string | string[], select?: string | string[] }) {
+        const queryOptions: any = {
             filter: { _id: request.id }
-        });
+        };
+
+        // Support select for single-record retrieval
+        if (request.select) {
+            queryOptions.select = typeof request.select === 'string'
+                ? request.select.split(',').map((s: string) => s.trim()).filter(Boolean)
+                : request.select;
+        }
+
+        // Support expand for single-record retrieval
+        if (request.expand) {
+            queryOptions.populate = typeof request.expand === 'string'
+                ? request.expand.split(',').map((s: string) => s.trim()).filter(Boolean)
+                : request.expand;
+        }
+
+        const result = await this.engine.findOne(request.object, queryOptions);
         if (result) {
             return {
                 object: request.object,