@objectstack/objectql 4.0.1 → 4.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@objectstack/objectql",
-  "version": "4.0.1",
+  "version": "4.0.3",
   "license": "Apache-2.0",
   "description": "Isomorphic ObjectQL Engine for ObjectStack",
   "main": "dist/index.js",
@@ -13,13 +13,13 @@
     }
   },
   "dependencies": {
-    "@objectstack/core": "4.0.1",
-    "@objectstack/spec": "4.0.1",
-    "@objectstack/types": "4.0.1"
+    "@objectstack/core": "4.0.3",
+    "@objectstack/spec": "4.0.3",
+    "@objectstack/types": "4.0.3"
   },
   "devDependencies": {
     "typescript": "^6.0.2",
-    "vitest": "^4.1.2"
+    "vitest": "^4.1.4"
   },
   "scripts": {
     "build": "tsup --config ../../tsup.config.ts",

package/src/engine.ts

@@ -1,17 +1,19 @@
 // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
 
 import { QueryAST, HookContext, ServiceObject } from '@objectstack/spec/data';
-import { 
+import {
   EngineQueryOptions,
-  DataEngineInsertOptions, 
-  EngineUpdateOptions, 
+  DataEngineInsertOptions,
+  EngineUpdateOptions,
   EngineDeleteOptions,
   EngineAggregateOptions,
-  EngineCountOptions 
+  EngineCountOptions
 } from '@objectstack/spec/data';
 import { ExecutionContext, ExecutionContextSchema } from '@objectstack/spec/kernel';
 import { DriverInterface, IDataEngine, Logger, createLogger } from '@objectstack/core';
 import { CoreServiceName } from '@objectstack/spec/system';
+import { IRealtimeService, RealtimeEventPayload } from '@objectstack/spec/contracts';
+import { pluralToSingular } from '@objectstack/spec/shared';
 import { SchemaRegistry } from './registry.js';
 
 export type HookHandler = (context: HookContext) => Promise<void> | void;
@@ -69,7 +71,7 @@ export class ObjectQL implements IDataEngine {
   private drivers = new Map<string, DriverInterface>();
   private defaultDriver: string | null = null;
   private logger: Logger;
-  
+
   // Per-object hooks with priority support
   private hooks: Map<string, HookEntry[]> = new Map([
     ['beforeFind', []], ['afterFind', []],
@@ -86,10 +88,13 @@ export class ObjectQL implements IDataEngine {
 
   // Action registry: key = "objectName:actionName"
   private actions = new Map<string, { handler: (ctx: any) => Promise<any> | any; package?: string }>();
-  
+
   // Host provided context additions (e.g. Server router)
   private hostContext: Record<string, any> = {};
 
+  // Realtime service for event publishing
+  private realtimeService?: IRealtimeService;
+
   constructor(hostContext: Record<string, any> = {}) {
     this.hostContext = hostContext;
     // Use provided logger or create a new one
@@ -391,7 +396,7 @@ export class ObjectQL implements IDataEngine {
               for (const item of items) {
                   const itemName = item.name || item.id;
                   if (itemName) {
-                      SchemaRegistry.registerItem(key, item, 'name' as any, id);
+                      SchemaRegistry.registerItem(pluralToSingular(key), item, 'name' as any, id);
                   }
               }
           }
@@ -497,7 +502,7 @@ export class ObjectQL implements IDataEngine {
               for (const item of items) {
                   const itemName = item.name || item.id;
                   if (itemName) {
-                      SchemaRegistry.registerItem(key, item, 'name' as any, ownerId);
+                      SchemaRegistry.registerItem(pluralToSingular(key), item, 'name' as any, ownerId);
                   }
               }
           }
@@ -514,8 +519,8 @@ export class ObjectQL implements IDataEngine {
     }
 
     this.drivers.set(driver.name, driver);
-    this.logger.info('Registered driver', { 
-      driverName: driver.name, 
+    this.logger.info('Registered driver', {
+      driverName: driver.name,
       version: driver.version
     });
 
@@ -525,6 +530,17 @@ export class ObjectQL implements IDataEngine {
     }
   }
 
+  /**
+   * Set the realtime service for publishing data change events.
+   * Should be called after kernel resolves the realtime service.
+   *
+   * @param service - An IRealtimeService instance for event publishing
+   */
+  setRealtimeService(service: IRealtimeService): void {
+    this.realtimeService = service;
+    this.logger.info('RealtimeService configured for data events');
+  }
+
   /**
    * Helper to get object definition
    */
@@ -594,14 +610,24 @@ export class ObjectQL implements IDataEngine {
       drivers: Array.from(this.drivers.keys())
     });
     
+    const failedDrivers: string[] = [];
     for (const [name, driver] of this.drivers) {
       try {
         await driver.connect();
         this.logger.info('Driver connected successfully', { driverName: name });
       } catch (e) {
+        failedDrivers.push(name);
         this.logger.error('Failed to connect driver', e as Error, { driverName: name });
       }
     }
+
+    if (failedDrivers.length > 0) {
+      this.logger.warn(
+        `${failedDrivers.length} of ${this.drivers.size} driver(s) failed initial connect. ` +
+        `Operations may recover via lazy reconnection or fail at query time.`,
+        { failedDrivers }
+      );
+    }
     
     this.logger.info('ObjectQL engine initialization complete');
   }
@@ -873,6 +899,42 @@ export class ObjectQL implements IDataEngine {
         hookContext.result = result;
         await this.triggerHooks('afterInsert', hookContext);
 
+        // Publish data.record.created event to realtime service
+        if (this.realtimeService) {
+          try {
+            if (Array.isArray(result)) {
+              // Bulk insert - publish event for each record
+              for (const record of result) {
+                const event: RealtimeEventPayload = {
+                  type: 'data.record.created',
+                  object,
+                  payload: {
+                    recordId: record.id,
+                    after: record,
+                  },
+                  timestamp: new Date().toISOString(),
+                };
+                await this.realtimeService.publish(event);
+              }
+              this.logger.debug(`Published ${result.length} data.record.created events`, { object });
+            } else {
+              const event: RealtimeEventPayload = {
+                type: 'data.record.created',
+                object,
+                payload: {
+                  recordId: result.id,
+                  after: result,
+                },
+                timestamp: new Date().toISOString(),
+              };
+              await this.realtimeService.publish(event);
+              this.logger.debug('Published data.record.created event', { object, recordId: result.id });
+            }
+          } catch (error) {
+            this.logger.warn('Failed to publish data event', { object, error });
+          }
+        }
+
         return hookContext.result;
       } catch (e) {
         this.logger.error('Insert operation failed', e as Error, { object });
@@ -927,6 +989,29 @@ export class ObjectQL implements IDataEngine {
            hookContext.event = 'afterUpdate';
            hookContext.result = result;
            await this.triggerHooks('afterUpdate', hookContext);
+
+           // Publish data.record.updated event to realtime service
+           if (this.realtimeService) {
+             try {
+               const resultId = (typeof result === 'object' && result && 'id' in result) ? (result as any).id : undefined;
+               const recordId = String(hookContext.input.id || resultId || '');
+               const event: RealtimeEventPayload = {
+                 type: 'data.record.updated',
+                 object,
+                 payload: {
+                   recordId,
+                   changes: hookContext.input.data,
+                   after: result,
+                 },
+                 timestamp: new Date().toISOString(),
+               };
+               await this.realtimeService.publish(event);
+               this.logger.debug('Published data.record.updated event', { object, recordId });
+             } catch (error) {
+               this.logger.warn('Failed to publish data event', { object, error });
+             }
+           }
+
            return hookContext.result;
        } catch (e) {
           this.logger.error('Update operation failed', e as Error, { object });
@@ -980,6 +1065,27 @@ export class ObjectQL implements IDataEngine {
           hookContext.event = 'afterDelete';
           hookContext.result = result;
           await this.triggerHooks('afterDelete', hookContext);
+
+          // Publish data.record.deleted event to realtime service
+          if (this.realtimeService) {
+            try {
+              const resultId = (typeof result === 'object' && result && 'id' in result) ? (result as any).id : undefined;
+              const recordId = String(hookContext.input.id || resultId || '');
+              const event: RealtimeEventPayload = {
+                type: 'data.record.deleted',
+                object,
+                payload: {
+                  recordId,
+                },
+                timestamp: new Date().toISOString(),
+              };
+              await this.realtimeService.publish(event);
+              this.logger.debug('Published data.record.deleted event', { object, recordId });
+            } catch (error) {
+              this.logger.warn('Failed to publish data event', { object, error });
+            }
+          }
+
           return hookContext.result;
       } catch (e) {
           this.logger.error('Delete operation failed', e as Error, { object });

package/src/plugin.integration.test.ts

@@ -15,7 +15,7 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
   });
 
   describe('Simple Mode (ObjectQL-only)', () => {
-    it('should register ObjectQL as metadata service provider', async () => {
+    it('should register objectql, data, and protocol services', async () => {
       // Arrange
       const plugin = new ObjectQLPlugin();
       await kernel.use(plugin);
@@ -23,16 +23,15 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
       // Act
       await kernel.bootstrap();
 
-      // Assert
-      const metadataService = kernel.getService('metadata');
-      expect(metadataService).toBeDefined();
-
-      // ObjectQL registers a MetadataFacade as the metadata service;
-      // it is separate from (but backed by the same registry as) the objectql service.
+      // Assert — ObjectQL no longer registers metadata (kernel provides fallback)
       const objectql = kernel.getService('objectql');
       expect(objectql).toBeDefined();
-      // metadata and objectql are distinct service instances
-      expect(metadataService).not.toBe(objectql);
+      expect(kernel.getService('data')).toBeDefined();
+      expect(kernel.getService('protocol')).toBeDefined();
+      // metadata is provided by kernel's core fallback, not ObjectQL
+      const metadataService = kernel.getService('metadata');
+      expect(metadataService).toBeDefined();
+      expect((metadataService as any)._fallback).toBe(true);
     });
 
     it('should serve in-memory metadata definitions', async () => {
@@ -65,7 +64,7 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
   });
 
   describe('Service Registration', () => {
-    it('should register objectql, data, and protocol services', async () => {
+    it('should register manifest service', async () => {
       // Arrange
       const plugin = new ObjectQLPlugin();
       await kernel.use(plugin);
@@ -77,6 +76,7 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
       expect(kernel.getService('objectql')).toBeDefined();
       expect(kernel.getService('data')).toBeDefined();
       expect(kernel.getService('protocol')).toBeDefined();
+      expect(kernel.getService('manifest')).toBeDefined();
     });
 
     it('should respect existing metadata service', async () => {
@@ -146,8 +146,71 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
       expect(objectql.drivers?.has('mock-driver')).toBe(true);
     });
 
-    it('should discover and register apps from kernel services', async () => {
+    it('should register apps via manifest service', async () => {
       // Arrange
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Plugin that uses the manifest service directly
+      await kernel.use({
+        name: 'mock-app-plugin',
+        type: 'app',
+        version: '1.0.0',
+        dependencies: ['com.objectstack.engine.objectql'],
+        init: async (ctx) => {
+          ctx.getService<{ register(m: any): void }>('manifest').register({
+            id: 'test-app',
+            name: 'test_app',
+            version: '1.0.0',
+            type: 'app',
+            apps: [{ name: 'Test App' }],
+          });
+        }
+      });
+
+      // Act
+      await kernel.bootstrap();
+
+      // Assert
+      const objectql = kernel.getService('objectql') as any;
+      expect(objectql.registry).toBeDefined();
+      const apps = objectql.registry.getAllApps();
+      expect(apps.some((a: any) => a.name === 'Test App')).toBe(true);
+    });
+
+    it('should register manifests from start() phase via manifest service', async () => {
+      // Arrange — simulates SetupPlugin's pattern (registers in start, not init)
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      await kernel.use({
+        name: 'late-registerer',
+        type: 'standard',
+        version: '1.0.0',
+        dependencies: ['com.objectstack.engine.objectql'],
+        init: async () => {},
+        start: async (ctx) => {
+          ctx.getService<{ register(m: any): void }>('manifest').register({
+            id: 'late-app',
+            name: 'late_app',
+            version: '1.0.0',
+            type: 'plugin',
+            apps: [{ name: 'Late App' }],
+          });
+        }
+      });
+
+      // Act
+      await kernel.bootstrap();
+
+      // Assert
+      const objectql = kernel.getService('objectql') as any;
+      const apps = objectql.registry.getAllApps();
+      expect(apps.some((a: any) => a.name === 'Late App')).toBe(true);
+    });
+
+    it('should still discover apps registered via legacy app.* convention', async () => {
+      // Arrange — legacy pattern for backward compatibility
       const mockApp = {
         manifest: {
           id: 'test-app',
@@ -172,9 +235,8 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
       // Act
       await kernel.bootstrap();
 
-      // Assert
+      // Assert — legacy pattern still works
       const objectql = kernel.getService('objectql') as any;
-      // App should be registered (check via registry or apps list)
       expect(objectql.registry).toBeDefined();
     });
   });
@@ -760,4 +822,174 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
       expect(singleCalls).toContain('nb__item');
     });
   });
+
+  describe('Cold-Start Metadata Restoration', () => {
+    it('should restore metadata from sys_metadata via protocol.loadMetaFromDb on start', async () => {
+      // Arrange — a driver whose find() returns persisted metadata records
+      const findCalls: Array<{ object: string; query: any }> = [];
+      const mockDriver = {
+        name: 'restore-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string, query: any) => {
+          findCalls.push({ object, query });
+          if (object === 'sys_metadata') {
+            return [
+              {
+                id: '1',
+                type: 'apps',
+                name: 'custom_crm',
+                state: 'active',
+                metadata: JSON.stringify({ name: 'custom_crm', label: 'Custom CRM' }),
+              },
+              {
+                id: '2',
+                type: 'object',
+                name: 'invoice',
+                state: 'active',
+                metadata: JSON.stringify({
+                  name: 'invoice',
+                  label: 'Invoice',
+                  fields: { amount: { name: 'amount', label: 'Amount', type: 'number' } },
+                }),
+                packageId: 'user_pkg',
+              },
+            ];
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async () => {},
+      };
+
+      await kernel.use({
+        name: 'mock-restore-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.restore', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act
+      await kernel.bootstrap();
+
+      // Assert — sys_metadata should have been queried
+      const metaQuery = findCalls.find((c) => c.object === 'sys_metadata');
+      expect(metaQuery).toBeDefined();
+      expect(metaQuery!.query.where).toEqual({ state: 'active' });
+
+      // Assert — items should be restored into the registry
+      const registry = (kernel.getService('objectql') as any).registry;
+      expect(registry.getAllApps()).toContainEqual({
+        name: 'custom_crm',
+        label: 'Custom CRM',
+      });
+    });
+
+    it('should not throw when protocol.loadMetaFromDb fails (graceful degradation)', async () => {
+      // Arrange — driver that throws on find('sys_metadata')
+      const mockDriver = {
+        name: 'failing-db-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string) => {
+          if (object === 'sys_metadata') {
+            throw new Error('SQLITE_ERROR: no such table: sys_metadata');
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async () => {},
+      };
+
+      await kernel.use({
+        name: 'mock-fail-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.faildb', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act & Assert — should not throw
+      await expect(kernel.bootstrap()).resolves.not.toThrow();
+    });
+
+    it('should restore metadata before syncRegisteredSchemas so restored objects get table sync', async () => {
+      // Arrange — track the order of operations
+      const operations: string[] = [];
+      const mockDriver = {
+        name: 'order-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string) => {
+          if (object === 'sys_metadata') {
+            operations.push('loadMetaFromDb');
+            return [
+              {
+                id: '1',
+                type: 'object',
+                name: 'restored_obj',
+                state: 'active',
+                metadata: JSON.stringify({
+                  name: 'restored_obj',
+                  label: 'Restored Object',
+                  fields: { title: { name: 'title', label: 'Title', type: 'text' } },
+                }),
+                packageId: 'user_pkg',
+              },
+            ];
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async (object: string) => {
+          operations.push(`syncSchema:${object}`);
+        },
+      };
+
+      await kernel.use({
+        name: 'mock-order-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.order', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act
+      await kernel.bootstrap();
+
+      // Assert — loadMetaFromDb must appear before any syncSchema call
+      const loadIdx = operations.indexOf('loadMetaFromDb');
+      expect(loadIdx).toBeGreaterThanOrEqual(0);
+
+      const firstSync = operations.findIndex((op) => op.startsWith('syncSchema:'));
+      if (firstSync >= 0) {
+        expect(loadIdx).toBeLessThan(firstSync);
+      }
+    });
+  });
 });