@objectstack/objectql 4.0.2 → 4.0.4

package/src/plugin.integration.test.ts

@@ -822,4 +822,174 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
       expect(singleCalls).toContain('nb__item');
     });
   });
+
+  describe('Cold-Start Metadata Restoration', () => {
+    it('should restore metadata from sys_metadata via protocol.loadMetaFromDb on start', async () => {
+      // Arrange — a driver whose find() returns persisted metadata records
+      const findCalls: Array<{ object: string; query: any }> = [];
+      const mockDriver = {
+        name: 'restore-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string, query: any) => {
+          findCalls.push({ object, query });
+          if (object === 'sys_metadata') {
+            return [
+              {
+                id: '1',
+                type: 'apps',
+                name: 'custom_crm',
+                state: 'active',
+                metadata: JSON.stringify({ name: 'custom_crm', label: 'Custom CRM' }),
+              },
+              {
+                id: '2',
+                type: 'object',
+                name: 'invoice',
+                state: 'active',
+                metadata: JSON.stringify({
+                  name: 'invoice',
+                  label: 'Invoice',
+                  fields: { amount: { name: 'amount', label: 'Amount', type: 'number' } },
+                }),
+                packageId: 'user_pkg',
+              },
+            ];
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async () => {},
+      };
+
+      await kernel.use({
+        name: 'mock-restore-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.restore', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act
+      await kernel.bootstrap();
+
+      // Assert — sys_metadata should have been queried
+      const metaQuery = findCalls.find((c) => c.object === 'sys_metadata');
+      expect(metaQuery).toBeDefined();
+      expect(metaQuery!.query.where).toEqual({ state: 'active' });
+
+      // Assert — items should be restored into the registry
+      const registry = (kernel.getService('objectql') as any).registry;
+      expect(registry.getAllApps()).toContainEqual({
+        name: 'custom_crm',
+        label: 'Custom CRM',
+      });
+    });
+
+    it('should not throw when protocol.loadMetaFromDb fails (graceful degradation)', async () => {
+      // Arrange — driver that throws on find('sys_metadata')
+      const mockDriver = {
+        name: 'failing-db-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string) => {
+          if (object === 'sys_metadata') {
+            throw new Error('SQLITE_ERROR: no such table: sys_metadata');
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async () => {},
+      };
+
+      await kernel.use({
+        name: 'mock-fail-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.faildb', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act & Assert — should not throw
+      await expect(kernel.bootstrap()).resolves.not.toThrow();
+    });
+
+    it('should restore metadata before syncRegisteredSchemas so restored objects get table sync', async () => {
+      // Arrange — track the order of operations
+      const operations: string[] = [];
+      const mockDriver = {
+        name: 'order-driver',
+        version: '1.0.0',
+        connect: async () => {},
+        disconnect: async () => {},
+        find: async (object: string) => {
+          if (object === 'sys_metadata') {
+            operations.push('loadMetaFromDb');
+            return [
+              {
+                id: '1',
+                type: 'object',
+                name: 'restored_obj',
+                state: 'active',
+                metadata: JSON.stringify({
+                  name: 'restored_obj',
+                  label: 'Restored Object',
+                  fields: { title: { name: 'title', label: 'Title', type: 'text' } },
+                }),
+                packageId: 'user_pkg',
+              },
+            ];
+          }
+          return [];
+        },
+        findOne: async () => null,
+        create: async (_o: string, d: any) => d,
+        update: async (_o: string, _i: any, d: any) => d,
+        delete: async () => true,
+        syncSchema: async (object: string) => {
+          operations.push(`syncSchema:${object}`);
+        },
+      };
+
+      await kernel.use({
+        name: 'mock-order-driver',
+        type: 'driver',
+        version: '1.0.0',
+        init: async (ctx) => {
+          ctx.registerService('driver.order', mockDriver);
+        },
+      });
+
+      const plugin = new ObjectQLPlugin();
+      await kernel.use(plugin);
+
+      // Act
+      await kernel.bootstrap();
+
+      // Assert — loadMetaFromDb must appear before any syncSchema call
+      const loadIdx = operations.indexOf('loadMetaFromDb');
+      expect(loadIdx).toBeGreaterThanOrEqual(0);
+
+      const firstSync = operations.findIndex((op) => op.startsWith('syncSchema:'));
+      if (firstSync >= 0) {
+        expect(loadIdx).toBeLessThan(firstSync);
+      }
+    });
+  });
 });

package/src/plugin.ts

@@ -6,6 +6,25 @@ import { Plugin, PluginContext } from '@objectstack/core';
 
 export type { Plugin, PluginContext };
 
+/**
+ * Protocol extension for DB-based metadata hydration.
+ * `loadMetaFromDb` is implemented by ObjectStackProtocolImplementation but
+ * is NOT (yet) part of the canonical ObjectStackProtocol wire-contract in
+ * `@objectstack/spec`, since it is a server-side bootstrap concern only.
+ */
+interface ProtocolWithDbRestore {
+  loadMetaFromDb(): Promise<{ loaded: number; errors: number }>;
+}
+
+/** Type guard — checks whether the service exposes `loadMetaFromDb`. */
+function hasLoadMetaFromDb(service: unknown): service is ProtocolWithDbRestore {
+  return (
+    typeof service === 'object' &&
+    service !== null &&
+    typeof (service as Record<string, unknown>)['loadMetaFromDb'] === 'function'
+  );
+}
+
 export class ObjectQLPlugin implements Plugin {
   name = 'com.objectstack.engine.objectql';
   type = 'objectql';
@@ -94,22 +113,47 @@ export class ObjectQLPlugin implements Plugin {
                 ctx.logger.debug('Discovered and registered app service (legacy)', { serviceName: name });
             }
         }
+
+        // Bridge realtime service from kernel service registry to ObjectQL.
+        // RealtimeServicePlugin registers as 'realtime' service during init().
+        // This enables ObjectQL to publish data change events.
+        try {
+            const realtimeService = ctx.getService('realtime');
+            if (realtimeService && typeof realtimeService === 'object' && 'publish' in realtimeService) {
+                ctx.logger.info('[ObjectQLPlugin] Bridging realtime service to ObjectQL for event publishing');
+                this.ql.setRealtimeService(realtimeService as any);
+            }
+        } catch (e: any) {
+            ctx.logger.debug('[ObjectQLPlugin] No realtime service found — data events will not be published', {
+                error: e.message,
+            });
+        }
     }
 
     // Initialize drivers (calls driver.connect() which sets up persistence)
     await this.ql?.init();
 
+    // Restore persisted metadata from sys_metadata table.
+    // This hydrates SchemaRegistry with objects/views/apps that were saved
+    // via protocol.saveMetaItem() in a previous session, ensuring custom
+    // schemas survive cold starts and redeployments.
+    await this.restoreMetadataFromDb(ctx);
+
     // Sync all registered object schemas to database
     // This ensures tables/collections are created or updated for every
     // object registered by plugins (e.g., sys_user from plugin-auth).
     await this.syncRegisteredSchemas(ctx);
 
+    // Bridge all SchemaRegistry objects to metadata service
+    // This ensures AI tools and other IMetadataService consumers can see all objects
+    await this.bridgeObjectsToMetadataService(ctx);
+
     // Register built-in audit hooks
     this.registerAuditHooks(ctx);
 
     // Register tenant isolation middleware
     this.registerTenantMiddleware(ctx);
-    
+
     ctx.logger.info('ObjectQL engine started', {
         driversRegistered: this.ql?.['drivers']?.size || 0,
         objectsRegistered: this.ql?.registry?.getAllObjects?.()?.length || 0
@@ -332,6 +376,110 @@ export class ObjectQLPlugin implements Plugin {
     }
   }
 
+  /**
+   * Restore persisted metadata from the database (sys_metadata) on startup.
+   *
+   * Calls `protocol.loadMetaFromDb()` to bulk-load all active metadata
+   * records (objects, views, apps, etc.) into the in-memory SchemaRegistry.
+   * This closes the persistence loop so that user-created schemas survive
+   * kernel cold starts and redeployments.
+   *
+   * Gracefully degrades when:
+   * - The protocol service is unavailable (e.g., in-memory-only mode).
+   * - `loadMetaFromDb` is not implemented by the protocol shim.
+   * - The underlying driver/table does not exist yet (first-run scenario).
+   */
+  private async restoreMetadataFromDb(ctx: PluginContext): Promise<void> {
+    // Phase 1: Resolve protocol service (separate from DB I/O for clearer diagnostics)
+    let protocol: ProtocolWithDbRestore;
+    try {
+      const service = ctx.getService('protocol');
+      if (!service || !hasLoadMetaFromDb(service)) {
+        ctx.logger.debug('Protocol service does not support loadMetaFromDb, skipping DB restore');
+        return;
+      }
+      protocol = service;
+    } catch (e: unknown) {
+      ctx.logger.debug('Protocol service unavailable, skipping DB restore', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+      return;
+    }
+
+    // Phase 2: DB hydration (loads into SchemaRegistry)
+    try {
+      const { loaded, errors } = await protocol.loadMetaFromDb();
+
+      if (loaded > 0 || errors > 0) {
+        ctx.logger.info('Metadata restored from database to SchemaRegistry', { loaded, errors });
+      } else {
+        ctx.logger.debug('No persisted metadata found in database');
+      }
+    } catch (e: unknown) {
+      // Non-fatal: first-run or in-memory driver may not have sys_metadata yet
+      ctx.logger.debug('DB metadata restore failed (non-fatal)', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+    }
+  }
+
+  /**
+   * Bridge all SchemaRegistry objects to the metadata service.
+   *
+   * This ensures objects registered by plugins and loaded from sys_metadata
+   * are visible to AI tools and other consumers that query IMetadataService.
+   *
+   * Runs after both restoreMetadataFromDb() and syncRegisteredSchemas() to
+   * catch all objects in the SchemaRegistry regardless of their source.
+   */
+  private async bridgeObjectsToMetadataService(ctx: PluginContext): Promise<void> {
+    try {
+      const metadataService = ctx.getService<any>('metadata');
+      if (!metadataService || typeof metadataService.register !== 'function') {
+        ctx.logger.debug('Metadata service unavailable for bridging, skipping');
+        return;
+      }
+
+      if (!this.ql?.registry) {
+        ctx.logger.debug('SchemaRegistry unavailable for bridging, skipping');
+        return;
+      }
+
+      const objects = this.ql.registry.getAllObjects();
+      let bridged = 0;
+
+      for (const obj of objects) {
+        try {
+          // Check if object is already in metadata service to avoid duplicates
+          const existing = await metadataService.getObject(obj.name);
+          if (!existing) {
+            // Register object that exists in SchemaRegistry but not in metadata service
+            await metadataService.register('object', obj.name, obj);
+            bridged++;
+          }
+        } catch (e: unknown) {
+          ctx.logger.debug('Failed to bridge object to metadata service', {
+            object: obj.name,
+            error: e instanceof Error ? e.message : String(e),
+          });
+        }
+      }
+
+      if (bridged > 0) {
+        ctx.logger.info('Bridged objects from SchemaRegistry to metadata service', {
+          count: bridged,
+          total: objects.length
+        });
+      } else {
+        ctx.logger.debug('No objects needed bridging (all already in metadata service)');
+      }
+    } catch (e: unknown) {
+      ctx.logger.debug('Failed to bridge objects to metadata service', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+    }
+  }
+
   /**
    * Load metadata from external metadata service into ObjectQL registry
    * This enables ObjectQL to use file-based or remote metadata

package/src/protocol.ts

@@ -2,15 +2,16 @@
 
 import { ObjectStackProtocol } from '@objectstack/spec/api';
 import { IDataEngine } from '@objectstack/core';
-import type { 
-    BatchUpdateRequest, 
-    BatchUpdateResponse, 
+import type {
+    BatchUpdateRequest,
+    BatchUpdateResponse,
     UpdateManyDataRequest,
     DeleteManyDataRequest
 } from '@objectstack/spec/api';
 import type { MetadataCacheRequest, MetadataCacheResponse, ServiceInfo, ApiRoutes, WellKnownCapabilities } from '@objectstack/spec/api';
 import type { IFeedService } from '@objectstack/spec/contracts';
 import { parseFilterAST, isFilterAST } from '@objectstack/spec/data';
+import { PLURAL_TO_SINGULAR, SINGULAR_TO_PLURAL } from '@objectstack/spec/shared';
 
 // We import SchemaRegistry directly since this class lives in the same package
 import { SchemaRegistry } from './registry.js';
@@ -180,18 +181,31 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
     }
 
     async getMetaTypes() {
-        return {
-            types: SchemaRegistry.getRegisteredTypes()
-        };
+        const schemaTypes = SchemaRegistry.getRegisteredTypes();
+
+        // Also include types from MetadataService (runtime-registered: agent, tool, etc.)
+        let runtimeTypes: string[] = [];
+        try {
+            const services = this.getServicesRegistry?.();
+            const metadataService = services?.get('metadata');
+            if (metadataService && typeof metadataService.getRegisteredTypes === 'function') {
+                runtimeTypes = await metadataService.getRegisteredTypes();
+            }
+        } catch {
+            // MetadataService not available
+        }
+
+        const allTypes = Array.from(new Set([...schemaTypes, ...runtimeTypes]));
+        return { types: allTypes };
     }
 
     async getMetaItems(request: { type: string; packageId?: string }) {
         const { packageId } = request;
         let items = SchemaRegistry.listItems(request.type, packageId);
-        // Normalize singular/plural: REST uses singular ('app') but registry may store as plural ('apps')
+        // Normalize singular/plural using explicit mapping
         if (items.length === 0) {
-            const alt = request.type.endsWith('s') ? request.type.slice(0, -1) : request.type + 's';
-            items = SchemaRegistry.listItems(alt, packageId);
+            const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+            if (alt) items = SchemaRegistry.listItems(alt, packageId);
         }
 
         // Fallback to database if registry is empty for this type
@@ -212,8 +226,9 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                         return data;
                     });
                 } else {
-                    // Try alternate type name in DB
-                    const alt = request.type.endsWith('s') ? request.type.slice(0, -1) : request.type + 's';
+                    // Try alternate type name in DB using explicit mapping
+                    const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+                    if (alt) {
                     const altRecords = await this.engine.find('sys_metadata', {
                         where: { type: alt, state: 'active' }
                     });
@@ -226,12 +241,48 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                             return data;
                         });
                     }
+                    }
                 }
             } catch {
                 // DB not available, return registry results (empty)
             }
         }
 
+        // Merge with MetadataService (runtime-registered items: agents, tools, etc.)
+        try {
+            const services = this.getServicesRegistry?.();
+            const metadataService = services?.get('metadata');
+            if (metadataService && typeof metadataService.list === 'function') {
+                let runtimeItems = await metadataService.list(request.type);
+                // When filtering by packageId, only include runtime items that
+                // belong to the requested package. MetadataService.list() returns
+                // items from ALL packages, so we must filter here to respect the
+                // package scope requested by the caller (e.g., Studio sidebar).
+                if (packageId && runtimeItems && runtimeItems.length > 0) {
+                    runtimeItems = runtimeItems.filter((item: any) => item?._packageId === packageId);
+                }
+                if (runtimeItems && runtimeItems.length > 0) {
+                    // Merge, avoiding duplicates by name
+                    const itemMap = new Map<string, any>();
+                    for (const item of items) {
+                        const entry = item as any;
+                        if (entry && typeof entry === 'object' && 'name' in entry) {
+                            itemMap.set(entry.name, entry);
+                        }
+                    }
+                    for (const item of runtimeItems) {
+                        const entry = item as any;
+                        if (entry && typeof entry === 'object' && 'name' in entry) {
+                            itemMap.set(entry.name, entry);
+                        }
+                    }
+                    items = Array.from(itemMap.values());
+                }
+            }
+        } catch {
+            // MetadataService not available or doesn't support this type
+        }
+
         return {
             type: request.type,
             items
@@ -240,10 +291,10 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
 
     async getMetaItem(request: { type: string, name: string, packageId?: string }) {
         let item = SchemaRegistry.getItem(request.type, request.name);
-        // Normalize singular/plural
+        // Normalize singular/plural using explicit mapping
         if (item === undefined) {
-            const alt = request.type.endsWith('s') ? request.type.slice(0, -1) : request.type + 's';
-            item = SchemaRegistry.getItem(alt, request.name);
+            const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+            if (alt) item = SchemaRegistry.getItem(alt, request.name);
         }
 
         // Fallback to database if not in registry
@@ -259,8 +310,9 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                     // Hydrate back into registry for next time
                     SchemaRegistry.registerItem(request.type, item, 'name' as any);
                 } else {
-                    // Try alternate type name
-                    const alt = request.type.endsWith('s') ? request.type.slice(0, -1) : request.type + 's';
+                    // Try alternate type name using explicit mapping
+                    const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+                    if (alt) {
                     const altRecord = await this.engine.findOne('sys_metadata', {
                         where: { type: alt, name: request.name, state: 'active' }
                     });
@@ -271,12 +323,26 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                         // Hydrate back into registry for next time
                         SchemaRegistry.registerItem(request.type, item, 'name' as any);
                     }
+                    }
                 }
             } catch {
                 // DB not available, return undefined
             }
         }
 
+        // Fallback to MetadataService for runtime-registered items (agents, tools, etc.)
+        if (item === undefined) {
+            try {
+                const services = this.getServicesRegistry?.();
+                const metadataService = services?.get('metadata');
+                if (metadataService && typeof metadataService.get === 'function') {
+                    item = await metadataService.get(request.type, request.name);
+                }
+            } catch {
+                // MetadataService not available
+            }
+        }
+
         return {
             type: request.type,
             name: request.name,
@@ -563,7 +629,27 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
 
     async getMetaItemCached(request: { type: string, name: string, cacheRequest?: MetadataCacheRequest }): Promise<MetadataCacheResponse> {
         try {
-            const item = SchemaRegistry.getItem(request.type, request.name);
+            let item = SchemaRegistry.getItem(request.type, request.name);
+
+            // Normalize singular/plural using explicit mapping
+            if (!item) {
+                const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+                if (alt) item = SchemaRegistry.getItem(alt, request.name);
+            }
+
+            // Fallback to MetadataService (e.g. agents, tools registered in MetadataManager)
+            if (!item) {
+                try {
+                    const services = this.getServicesRegistry?.();
+                    const metadataService = services?.get('metadata');
+                    if (metadataService && typeof metadataService.get === 'function') {
+                        item = await metadataService.get(request.type, request.name);
+                    }
+                } catch {
+                    // MetadataService not available
+                }
+            }
+
             if (!item) {
                 throw new Error(`Metadata item ${request.type}/${request.name} not found`);
             }
@@ -984,10 +1070,12 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                     const data = typeof record.metadata === 'string'
                         ? JSON.parse(record.metadata)
                         : record.metadata;
-                    if (record.type === 'object') {
+                    // Normalize DB type to singular (DB may store legacy plural forms)
+                    const normalizedType = PLURAL_TO_SINGULAR[record.type] ?? record.type;
+                    if (normalizedType === 'object') {
                         SchemaRegistry.registerObject(data as any, record.packageId || 'sys_metadata');
                     } else {
-                        SchemaRegistry.registerItem(record.type, data, 'name' as any);
+                        SchemaRegistry.registerItem(normalizedType, data, 'name' as any);
                     }
                     loaded++;
                 } catch (e) {

package/src/registry.test.ts

@@ -353,17 +353,17 @@ describe('SchemaRegistry', () => {
     describe('Generic Metadata', () => {
         it('should register and retrieve generic items', () => {
             const item = { name: 'test_action', type: 'custom' };
-            SchemaRegistry.registerItem('actions', item, 'name', 'com.pkg');
-            
-            const retrieved = SchemaRegistry.getItem('actions', 'test_action');
+            SchemaRegistry.registerItem('action', item, 'name', 'com.pkg');
+
+            const retrieved = SchemaRegistry.getItem('action', 'test_action');
             expect(retrieved).toEqual(item);
         });
 
         it('should list items by type with package filter', () => {
-            SchemaRegistry.registerItem('actions', { name: 'a1' }, 'name', 'com.pkg1');
-            SchemaRegistry.registerItem('actions', { name: 'a2' }, 'name', 'com.pkg2');
-            
-            const filtered = SchemaRegistry.listItems('actions', 'com.pkg1');
+            SchemaRegistry.registerItem('action', { name: 'a1' }, 'name', 'com.pkg1');
+            SchemaRegistry.registerItem('action', { name: 'a2' }, 'name', 'com.pkg2');
+
+            const filtered = SchemaRegistry.listItems('action', 'com.pkg1');
             expect(filtered).toHaveLength(1);
         });
     });
@@ -396,12 +396,12 @@ describe('SchemaRegistry', () => {
     describe('Reset', () => {
         it('should clear all state', () => {
             SchemaRegistry.registerObject({ name: 'obj', fields: {} } as any, 'com.pkg', 'pkg', 'own');
-            SchemaRegistry.registerItem('actions', { name: 'act' }, 'name');
-            
+            SchemaRegistry.registerItem('action', { name: 'act' }, 'name');
+
             SchemaRegistry.reset();
-            
+
             expect(SchemaRegistry.getAllObjects()).toHaveLength(0);
-            expect(SchemaRegistry.listItems('actions')).toHaveLength(0);
+            expect(SchemaRegistry.listItems('action')).toHaveLength(0);
         });
     });
 

package/src/registry.ts

@@ -485,7 +485,7 @@ export class SchemaRegistry {
     if (type === 'object') {
       return ObjectSchema.parse(item);
     }
-    if (type === 'apps') {
+    if (type === 'app') {
       return AppSchema.parse(item);
     }
     if (type === 'package') {
@@ -664,15 +664,15 @@ export class SchemaRegistry {
   // ==========================================
 
   static registerApp(app: any, packageId?: string) {
-    this.registerItem('apps', app, 'name', packageId);
+    this.registerItem('app', app, 'name', packageId);
   }
 
   static getApp(name: string): any {
-    return this.getItem('apps', name);
+    return this.getItem('app', name);
   }
 
   static getAllApps(): any[] {
-    return this.listItems('apps');
+    return this.listItems('app');
   }
 
   // ==========================================