@objectstack/objectql 4.0.2 → 4.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@objectstack/objectql",
-  "version": "4.0.2",
+  "version": "4.0.3",
   "license": "Apache-2.0",
   "description": "Isomorphic ObjectQL Engine for ObjectStack",
   "main": "dist/index.js",
@@ -13,13 +13,13 @@
     }
   },
   "dependencies": {
-    "@objectstack/core": "4.0.2",
-    "@objectstack/spec": "4.0.2",
-    "@objectstack/types": "4.0.2"
+    "@objectstack/core": "4.0.3",
+    "@objectstack/spec": "4.0.3",
+    "@objectstack/types": "4.0.3"
   },
   "devDependencies": {
     "typescript": "^6.0.2",
-    "vitest": "^4.1.2"
+    "vitest": "^4.1.4"
   },
   "scripts": {
     "build": "tsup --config ../../tsup.config.ts",

package/src/engine.ts

@@ -1,17 +1,19 @@
 // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
 
 import { QueryAST, HookContext, ServiceObject } from '@objectstack/spec/data';
-import { 
+import {
   EngineQueryOptions,
-  DataEngineInsertOptions, 
-  EngineUpdateOptions, 
+  DataEngineInsertOptions,
+  EngineUpdateOptions,
   EngineDeleteOptions,
   EngineAggregateOptions,
-  EngineCountOptions 
+  EngineCountOptions
 } from '@objectstack/spec/data';
 import { ExecutionContext, ExecutionContextSchema } from '@objectstack/spec/kernel';
 import { DriverInterface, IDataEngine, Logger, createLogger } from '@objectstack/core';
 import { CoreServiceName } from '@objectstack/spec/system';
+import { IRealtimeService, RealtimeEventPayload } from '@objectstack/spec/contracts';
+import { pluralToSingular } from '@objectstack/spec/shared';
 import { SchemaRegistry } from './registry.js';
 
 export type HookHandler = (context: HookContext) => Promise<void> | void;
@@ -69,7 +71,7 @@ export class ObjectQL implements IDataEngine {
   private drivers = new Map<string, DriverInterface>();
   private defaultDriver: string | null = null;
   private logger: Logger;
-  
+
   // Per-object hooks with priority support
   private hooks: Map<string, HookEntry[]> = new Map([
     ['beforeFind', []], ['afterFind', []],
@@ -86,10 +88,13 @@ export class ObjectQL implements IDataEngine {
 
   // Action registry: key = "objectName:actionName"
   private actions = new Map<string, { handler: (ctx: any) => Promise<any> | any; package?: string }>();
-  
+
   // Host provided context additions (e.g. Server router)
   private hostContext: Record<string, any> = {};
 
+  // Realtime service for event publishing
+  private realtimeService?: IRealtimeService;
+
   constructor(hostContext: Record<string, any> = {}) {
     this.hostContext = hostContext;
     // Use provided logger or create a new one
@@ -391,7 +396,7 @@ export class ObjectQL implements IDataEngine {
               for (const item of items) {
                   const itemName = item.name || item.id;
                   if (itemName) {
-                      SchemaRegistry.registerItem(key, item, 'name' as any, id);
+                      SchemaRegistry.registerItem(pluralToSingular(key), item, 'name' as any, id);
                   }
               }
           }
@@ -497,7 +502,7 @@ export class ObjectQL implements IDataEngine {
               for (const item of items) {
                   const itemName = item.name || item.id;
                   if (itemName) {
-                      SchemaRegistry.registerItem(key, item, 'name' as any, ownerId);
+                      SchemaRegistry.registerItem(pluralToSingular(key), item, 'name' as any, ownerId);
                   }
               }
           }
@@ -514,8 +519,8 @@ export class ObjectQL implements IDataEngine {
     }
 
     this.drivers.set(driver.name, driver);
-    this.logger.info('Registered driver', { 
-      driverName: driver.name, 
+    this.logger.info('Registered driver', {
+      driverName: driver.name,
       version: driver.version
     });
 
@@ -525,6 +530,17 @@ export class ObjectQL implements IDataEngine {
     }
   }
 
+  /**
+   * Set the realtime service for publishing data change events.
+   * Should be called after kernel resolves the realtime service.
+   *
+   * @param service - An IRealtimeService instance for event publishing
+   */
+  setRealtimeService(service: IRealtimeService): void {
+    this.realtimeService = service;
+    this.logger.info('RealtimeService configured for data events');
+  }
+
   /**
    * Helper to get object definition
    */
@@ -594,14 +610,24 @@ export class ObjectQL implements IDataEngine {
       drivers: Array.from(this.drivers.keys())
     });
     
+    const failedDrivers: string[] = [];
     for (const [name, driver] of this.drivers) {
       try {
         await driver.connect();
         this.logger.info('Driver connected successfully', { driverName: name });
       } catch (e) {
+        failedDrivers.push(name);
         this.logger.error('Failed to connect driver', e as Error, { driverName: name });
       }
     }
+
+    if (failedDrivers.length > 0) {
+      this.logger.warn(
+        `${failedDrivers.length} of ${this.drivers.size} driver(s) failed initial connect. ` +
+        `Operations may recover via lazy reconnection or fail at query time.`,
+        { failedDrivers }
+      );
+    }
     
     this.logger.info('ObjectQL engine initialization complete');
   }
@@ -873,6 +899,42 @@ export class ObjectQL implements IDataEngine {
         hookContext.result = result;
         await this.triggerHooks('afterInsert', hookContext);
 
+        // Publish data.record.created event to realtime service
+        if (this.realtimeService) {
+          try {
+            if (Array.isArray(result)) {
+              // Bulk insert - publish event for each record
+              for (const record of result) {
+                const event: RealtimeEventPayload = {
+                  type: 'data.record.created',
+                  object,
+                  payload: {
+                    recordId: record.id,
+                    after: record,
+                  },
+                  timestamp: new Date().toISOString(),
+                };
+                await this.realtimeService.publish(event);
+              }
+              this.logger.debug(`Published ${result.length} data.record.created events`, { object });
+            } else {
+              const event: RealtimeEventPayload = {
+                type: 'data.record.created',
+                object,
+                payload: {
+                  recordId: result.id,
+                  after: result,
+                },
+                timestamp: new Date().toISOString(),
+              };
+              await this.realtimeService.publish(event);
+              this.logger.debug('Published data.record.created event', { object, recordId: result.id });
+            }
+          } catch (error) {
+            this.logger.warn('Failed to publish data event', { object, error });
+          }
+        }
+
         return hookContext.result;
       } catch (e) {
         this.logger.error('Insert operation failed', e as Error, { object });
@@ -927,6 +989,29 @@ export class ObjectQL implements IDataEngine {
            hookContext.event = 'afterUpdate';
            hookContext.result = result;
            await this.triggerHooks('afterUpdate', hookContext);
+
+           // Publish data.record.updated event to realtime service
+           if (this.realtimeService) {
+             try {
+               const resultId = (typeof result === 'object' && result && 'id' in result) ? (result as any).id : undefined;
+               const recordId = String(hookContext.input.id || resultId || '');
+               const event: RealtimeEventPayload = {
+                 type: 'data.record.updated',
+                 object,
+                 payload: {
+                   recordId,
+                   changes: hookContext.input.data,
+                   after: result,
+                 },
+                 timestamp: new Date().toISOString(),
+               };
+               await this.realtimeService.publish(event);
+               this.logger.debug('Published data.record.updated event', { object, recordId });
+             } catch (error) {
+               this.logger.warn('Failed to publish data event', { object, error });
+             }
+           }
+
            return hookContext.result;
        } catch (e) {
           this.logger.error('Update operation failed', e as Error, { object });
@@ -980,6 +1065,27 @@ export class ObjectQL implements IDataEngine {
           hookContext.event = 'afterDelete';
           hookContext.result = result;
           await this.triggerHooks('afterDelete', hookContext);
+
+          // Publish data.record.deleted event to realtime service
+          if (this.realtimeService) {
+            try {
+              const resultId = (typeof result === 'object' && result && 'id' in result) ? (result as any).id : undefined;
+              const recordId = String(hookContext.input.id || resultId || '');
+              const event: RealtimeEventPayload = {
+                type: 'data.record.deleted',
+                object,
+                payload: {
+                  recordId,
+                },
+                timestamp: new Date().toISOString(),
+              };
+              await this.realtimeService.publish(event);
+              this.logger.debug('Published data.record.deleted event', { object, recordId });
+            } catch (error) {
+              this.logger.warn('Failed to publish data event', { object, error });
+            }
+          }
+
           return hookContext.result;
       } catch (e) {
           this.logger.error('Delete operation failed', e as Error, { object });

package/src/plugin.integration.test.ts

@@ -822,4 +822,174 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
       expect(singleCalls).toContain('nb__item');
     });
   });
+
+  describe('Cold-Start Metadata Restoration', () => {
+    it('should restore metadata from sys_metadata via protocol.loadMetaFromDb on start', async () => {
+      // Arrange — a driver whose find() returns persisted metadata records
+      const findCalls: Array<{ object: string; query: any }> = [];
+      const mockDriver = {
+        name: 'restore-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string, query: any) => {
+          findCalls.push({ object, query });
+          if (object === 'sys_metadata') {
+            return [
+              {
+                id: '1',
+                type: 'apps',
+                name: 'custom_crm',
+                state: 'active',
+                metadata: JSON.stringify({ name: 'custom_crm', label: 'Custom CRM' }),
+              },
+              {
+                id: '2',
+                type: 'object',
+                name: 'invoice',
+                state: 'active',
+                metadata: JSON.stringify({
+                  name: 'invoice',
+                  label: 'Invoice',
+                  fields: { amount: { name: 'amount', label: 'Amount', type: 'number' } },
+                }),
+                packageId: 'user_pkg',
+              },
+            ];
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async () => {},
+      };
+
+      await kernel.use({
+        name: 'mock-restore-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.restore', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act
+      await kernel.bootstrap();
+
+      // Assert — sys_metadata should have been queried
+      const metaQuery = findCalls.find((c) => c.object === 'sys_metadata');
+      expect(metaQuery).toBeDefined();
+      expect(metaQuery!.query.where).toEqual({ state: 'active' });
+
+      // Assert — items should be restored into the registry
+      const registry = (kernel.getService('objectql') as any).registry;
+      expect(registry.getAllApps()).toContainEqual({
+        name: 'custom_crm',
+        label: 'Custom CRM',
+      });
+    });
+
+    it('should not throw when protocol.loadMetaFromDb fails (graceful degradation)', async () => {
+      // Arrange — driver that throws on find('sys_metadata')
+      const mockDriver = {
+        name: 'failing-db-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string) => {
+          if (object === 'sys_metadata') {
+            throw new Error('SQLITE_ERROR: no such table: sys_metadata');
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async () => {},
+      };
+
+      await kernel.use({
+        name: 'mock-fail-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.faildb', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act & Assert — should not throw
+      await expect(kernel.bootstrap()).resolves.not.toThrow();
+    });
+
+    it('should restore metadata before syncRegisteredSchemas so restored objects get table sync', async () => {
+      // Arrange — track the order of operations
+      const operations: string[] = [];
+      const mockDriver = {
+        name: 'order-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string) => {
+          if (object === 'sys_metadata') {
+            operations.push('loadMetaFromDb');
+            return [
+              {
+                id: '1',
+                type: 'object',
+                name: 'restored_obj',
+                state: 'active',
+                metadata: JSON.stringify({
+                  name: 'restored_obj',
+                  label: 'Restored Object',
+                  fields: { title: { name: 'title', label: 'Title', type: 'text' } },
+                }),
+                packageId: 'user_pkg',
+              },
+            ];
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async (object: string) => {
+          operations.push(`syncSchema:${object}`);
+        },
+      };
+
+      await kernel.use({
+        name: 'mock-order-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.order', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act
+      await kernel.bootstrap();
+
+      // Assert — loadMetaFromDb must appear before any syncSchema call
+      const loadIdx = operations.indexOf('loadMetaFromDb');
+      expect(loadIdx).toBeGreaterThanOrEqual(0);
+
+      const firstSync = operations.findIndex((op) => op.startsWith('syncSchema:'));
+      if (firstSync >= 0) {
+        expect(loadIdx).toBeLessThan(firstSync);
+      }
+    });
+  });
 });

package/src/plugin.ts

@@ -6,6 +6,25 @@ import { Plugin, PluginContext } from '@objectstack/core';
 
 export type { Plugin, PluginContext };
 
+/**
+ * Protocol extension for DB-based metadata hydration.
+ * `loadMetaFromDb` is implemented by ObjectStackProtocolImplementation but
+ * is NOT (yet) part of the canonical ObjectStackProtocol wire-contract in
+ * `@objectstack/spec`, since it is a server-side bootstrap concern only.
+ */
+interface ProtocolWithDbRestore {
+  loadMetaFromDb(): Promise<{ loaded: number; errors: number }>;
+}
+
+/** Type guard — checks whether the service exposes `loadMetaFromDb`. */
+function hasLoadMetaFromDb(service: unknown): service is ProtocolWithDbRestore {
+  return (
+    typeof service === 'object' &&
+    service !== null &&
+    typeof (service as Record<string, unknown>)['loadMetaFromDb'] === 'function'
+  );
+}
+
 export class ObjectQLPlugin implements Plugin {
   name = 'com.objectstack.engine.objectql';
   type = 'objectql';
@@ -94,22 +113,47 @@ export class ObjectQLPlugin implements Plugin {
                 ctx.logger.debug('Discovered and registered app service (legacy)', { serviceName: name });
             }
         }
+
+        // Bridge realtime service from kernel service registry to ObjectQL.
+        // RealtimeServicePlugin registers as 'realtime' service during init().
+        // This enables ObjectQL to publish data change events.
+        try {
+            const realtimeService = ctx.getService('realtime');
+            if (realtimeService && typeof realtimeService === 'object' && 'publish' in realtimeService) {
+                ctx.logger.info('[ObjectQLPlugin] Bridging realtime service to ObjectQL for event publishing');
+                this.ql.setRealtimeService(realtimeService as any);
+            }
+        } catch (e: any) {
+            ctx.logger.debug('[ObjectQLPlugin] No realtime service found — data events will not be published', {
+                error: e.message,
+            });
+        }
     }
 
     // Initialize drivers (calls driver.connect() which sets up persistence)
     await this.ql?.init();
 
+    // Restore persisted metadata from sys_metadata table.
+    // This hydrates SchemaRegistry with objects/views/apps that were saved
+    // via protocol.saveMetaItem() in a previous session, ensuring custom
+    // schemas survive cold starts and redeployments.
+    await this.restoreMetadataFromDb(ctx);
+
     // Sync all registered object schemas to database
     // This ensures tables/collections are created or updated for every
     // object registered by plugins (e.g., sys_user from plugin-auth).
     await this.syncRegisteredSchemas(ctx);
 
+    // Bridge all SchemaRegistry objects to metadata service
+    // This ensures AI tools and other IMetadataService consumers can see all objects
+    await this.bridgeObjectsToMetadataService(ctx);
+
     // Register built-in audit hooks
     this.registerAuditHooks(ctx);
 
     // Register tenant isolation middleware
     this.registerTenantMiddleware(ctx);
-    
+
     ctx.logger.info('ObjectQL engine started', {
         driversRegistered: this.ql?.['drivers']?.size || 0,
         objectsRegistered: this.ql?.registry?.getAllObjects?.()?.length || 0
@@ -332,6 +376,110 @@ export class ObjectQLPlugin implements Plugin {
     }
   }
 
+  /**
+   * Restore persisted metadata from the database (sys_metadata) on startup.
+   *
+   * Calls `protocol.loadMetaFromDb()` to bulk-load all active metadata
+   * records (objects, views, apps, etc.) into the in-memory SchemaRegistry.
+   * This closes the persistence loop so that user-created schemas survive
+   * kernel cold starts and redeployments.
+   *
+   * Gracefully degrades when:
+   * - The protocol service is unavailable (e.g., in-memory-only mode).
+   * - `loadMetaFromDb` is not implemented by the protocol shim.
+   * - The underlying driver/table does not exist yet (first-run scenario).
+   */
+  private async restoreMetadataFromDb(ctx: PluginContext): Promise<void> {
+    // Phase 1: Resolve protocol service (separate from DB I/O for clearer diagnostics)
+    let protocol: ProtocolWithDbRestore;
+    try {
+      const service = ctx.getService('protocol');
+      if (!service || !hasLoadMetaFromDb(service)) {
+        ctx.logger.debug('Protocol service does not support loadMetaFromDb, skipping DB restore');
+        return;
+      }
+      protocol = service;
+    } catch (e: unknown) {
+      ctx.logger.debug('Protocol service unavailable, skipping DB restore', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+      return;
+    }
+
+    // Phase 2: DB hydration (loads into SchemaRegistry)
+    try {
+      const { loaded, errors } = await protocol.loadMetaFromDb();
+
+      if (loaded > 0 || errors > 0) {
+        ctx.logger.info('Metadata restored from database to SchemaRegistry', { loaded, errors });
+      } else {
+        ctx.logger.debug('No persisted metadata found in database');
+      }
+    } catch (e: unknown) {
+      // Non-fatal: first-run or in-memory driver may not have sys_metadata yet
+      ctx.logger.debug('DB metadata restore failed (non-fatal)', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+    }
+  }
+
+  /**
+   * Bridge all SchemaRegistry objects to the metadata service.
+   *
+   * This ensures objects registered by plugins and loaded from sys_metadata
+   * are visible to AI tools and other consumers that query IMetadataService.
+   *
+   * Runs after both restoreMetadataFromDb() and syncRegisteredSchemas() to
+   * catch all objects in the SchemaRegistry regardless of their source.
+   */
+  private async bridgeObjectsToMetadataService(ctx: PluginContext): Promise<void> {
+    try {
+      const metadataService = ctx.getService<any>('metadata');
+      if (!metadataService || typeof metadataService.register !== 'function') {
+        ctx.logger.debug('Metadata service unavailable for bridging, skipping');
+        return;
+      }
+
+      if (!this.ql?.registry) {
+        ctx.logger.debug('SchemaRegistry unavailable for bridging, skipping');
+        return;
+      }
+
+      const objects = this.ql.registry.getAllObjects();
+      let bridged = 0;
+
+      for (const obj of objects) {
+        try {
+          // Check if object is already in metadata service to avoid duplicates
+          const existing = await metadataService.getObject(obj.name);
+          if (!existing) {
+            // Register object that exists in SchemaRegistry but not in metadata service
+            await metadataService.register('object', obj.name, obj);
+            bridged++;
+          }
+        } catch (e: unknown) {
+          ctx.logger.debug('Failed to bridge object to metadata service', {
+            object: obj.name,
+            error: e instanceof Error ? e.message : String(e),
+          });
+        }
+      }
+
+      if (bridged > 0) {
+        ctx.logger.info('Bridged objects from SchemaRegistry to metadata service', {
+          count: bridged,
+          total: objects.length
+        });
+      } else {
+        ctx.logger.debug('No objects needed bridging (all already in metadata service)');
+      }
+    } catch (e: unknown) {
+      ctx.logger.debug('Failed to bridge objects to metadata service', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+    }
+  }
+
   /**
    * Load metadata from external metadata service into ObjectQL registry
    * This enables ObjectQL to use file-based or remote metadata