@objectstack/objectql 4.0.1 → 4.0.3

package/src/plugin.ts

@@ -1,12 +1,30 @@
 // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
 
 import { ObjectQL } from './engine.js';
-import { MetadataFacade } from './metadata-facade.js';
 import { ObjectStackProtocolImplementation } from './protocol.js';
 import { Plugin, PluginContext } from '@objectstack/core';
 
 export type { Plugin, PluginContext };
 
+/**
+ * Protocol extension for DB-based metadata hydration.
+ * `loadMetaFromDb` is implemented by ObjectStackProtocolImplementation but
+ * is NOT (yet) part of the canonical ObjectStackProtocol wire-contract in
+ * `@objectstack/spec`, since it is a server-side bootstrap concern only.
+ */
+interface ProtocolWithDbRestore {
+  loadMetaFromDb(): Promise<{ loaded: number; errors: number }>;
+}
+
+/** Type guard — checks whether the service exposes `loadMetaFromDb`. */
+function hasLoadMetaFromDb(service: unknown): service is ProtocolWithDbRestore {
+  return (
+    typeof service === 'object' &&
+    service !== null &&
+    typeof (service as Record<string, unknown>)['loadMetaFromDb'] === 'function'
+  );
+}
+
 export class ObjectQLPlugin implements Plugin {
   name = 'com.objectstack.engine.objectql';
   type = 'objectql';
@@ -33,45 +51,24 @@ export class ObjectQLPlugin implements Plugin {
     
     // Register as provider for Core Kernel Services
     ctx.registerService('objectql', this.ql);
-    
-    // Register MetadataFacade as metadata service (unless external service exists)
-    let hasMetadata = false;
-    let metadataProvider = 'objectql';
-    try {
-        if (ctx.getService('metadata')) {
-            hasMetadata = true;
-            metadataProvider = 'external';
-        }
-    } catch (e: any) {
-        // Ignore errors during check (e.g. "Service is async")
-    }
 
-    if (!hasMetadata) {
-        try {
-            const metadataFacade = new MetadataFacade();
-            ctx.registerService('metadata', metadataFacade);
-            ctx.logger.info('MetadataFacade registered as metadata service', {
-                mode: 'in-memory',
-                features: ['registry', 'fast-lookup']
-            });
-        } catch (e: any) {
-             // Ignore if already registered (race condition or async mis-detection)
-             if (!e.message?.includes('already registered')) {
-                 throw e;
-             }
-        }
-    } else {
-        ctx.logger.info('External metadata service detected', {
-            provider: metadataProvider,
-            mode: 'will-sync-in-start-phase'
-        });
-    }
-    
     ctx.registerService('data', this.ql); // ObjectQL implements IDataEngine
-    
-    ctx.logger.info('ObjectQL engine registered', { 
-        services: ['objectql', 'data'],
-        metadataProvider: metadataProvider
+
+    // Register manifest service for direct app/package registration.
+    // Plugins call ctx.getService('manifest').register(manifestData)
+    // instead of the legacy ctx.registerService('app.<id>', manifestData) convention.
+    const ql = this.ql;
+    ctx.registerService('manifest', {
+      register: (manifest: any) => {
+        ql.registerApp(manifest);
+        ctx.logger.debug('Manifest registered via manifest service', {
+          id: manifest.id || manifest.name
+        });
+      }
+    });
+
+    ctx.logger.info('ObjectQL engine registered', {
+        services: ['objectql', 'data', 'manifest'],
     });
 
     // Register Protocol Implementation
@@ -86,16 +83,14 @@ export class ObjectQLPlugin implements Plugin {
 
   start = async (ctx: PluginContext) => {
     ctx.logger.info('ObjectQL engine starting...');
-    
-    // Check if we should load from external metadata service
+
+    // Sync from external metadata service (e.g. MetadataPlugin) if available
     try {
         const metadataService = ctx.getService('metadata') as any;
-        // Only sync if metadata service is external (not our own MetadataFacade)
-        if (metadataService && !(metadataService instanceof MetadataFacade) && this.ql) {
+        if (metadataService && typeof metadataService.loadMany === 'function' && this.ql) {
             await this.loadMetadataFromService(metadataService, ctx);
         }
     } catch (e: any) {
-        // No external metadata service or error accessing it
         ctx.logger.debug('No external metadata service to sync from');
     }
     
@@ -109,27 +104,56 @@ export class ObjectQLPlugin implements Plugin {
                  ctx.logger.debug('Discovered and registered driver service', { serviceName: name });
             }
             if (name.startsWith('app.')) {
-                // Register App
+                // Legacy fallback: discover app.* services (DEPRECATED)
+                ctx.logger.warn(
+                    `[DEPRECATED] Service "${name}" uses legacy app.* convention. ` +
+                    `Migrate to ctx.getService('manifest').register(data).`
+                );
                 this.ql.registerApp(service); // service is Manifest
-                ctx.logger.debug('Discovered and registered app service', { serviceName: name });
+                ctx.logger.debug('Discovered and registered app service (legacy)', { serviceName: name });
+            }
+        }
+
+        // Bridge realtime service from kernel service registry to ObjectQL.
+        // RealtimeServicePlugin registers as 'realtime' service during init().
+        // This enables ObjectQL to publish data change events.
+        try {
+            const realtimeService = ctx.getService('realtime');
+            if (realtimeService && typeof realtimeService === 'object' && 'publish' in realtimeService) {
+                ctx.logger.info('[ObjectQLPlugin] Bridging realtime service to ObjectQL for event publishing');
+                this.ql.setRealtimeService(realtimeService as any);
             }
+        } catch (e: any) {
+            ctx.logger.debug('[ObjectQLPlugin] No realtime service found — data events will not be published', {
+                error: e.message,
+            });
         }
     }
 
     // Initialize drivers (calls driver.connect() which sets up persistence)
     await this.ql?.init();
 
+    // Restore persisted metadata from sys_metadata table.
+    // This hydrates SchemaRegistry with objects/views/apps that were saved
+    // via protocol.saveMetaItem() in a previous session, ensuring custom
+    // schemas survive cold starts and redeployments.
+    await this.restoreMetadataFromDb(ctx);
+
     // Sync all registered object schemas to database
     // This ensures tables/collections are created or updated for every
     // object registered by plugins (e.g., sys_user from plugin-auth).
     await this.syncRegisteredSchemas(ctx);
 
+    // Bridge all SchemaRegistry objects to metadata service
+    // This ensures AI tools and other IMetadataService consumers can see all objects
+    await this.bridgeObjectsToMetadataService(ctx);
+
     // Register built-in audit hooks
     this.registerAuditHooks(ctx);
 
     // Register tenant isolation middleware
     this.registerTenantMiddleware(ctx);
-    
+
     ctx.logger.info('ObjectQL engine started', {
         driversRegistered: this.ql?.['drivers']?.size || 0,
         objectsRegistered: this.ql?.registry?.getAllObjects?.()?.length || 0
@@ -352,6 +376,110 @@ export class ObjectQLPlugin implements Plugin {
     }
   }
 
+  /**
+   * Restore persisted metadata from the database (sys_metadata) on startup.
+   *
+   * Calls `protocol.loadMetaFromDb()` to bulk-load all active metadata
+   * records (objects, views, apps, etc.) into the in-memory SchemaRegistry.
+   * This closes the persistence loop so that user-created schemas survive
+   * kernel cold starts and redeployments.
+   *
+   * Gracefully degrades when:
+   * - The protocol service is unavailable (e.g., in-memory-only mode).
+   * - `loadMetaFromDb` is not implemented by the protocol shim.
+   * - The underlying driver/table does not exist yet (first-run scenario).
+   */
+  private async restoreMetadataFromDb(ctx: PluginContext): Promise<void> {
+    // Phase 1: Resolve protocol service (separate from DB I/O for clearer diagnostics)
+    let protocol: ProtocolWithDbRestore;
+    try {
+      const service = ctx.getService('protocol');
+      if (!service || !hasLoadMetaFromDb(service)) {
+        ctx.logger.debug('Protocol service does not support loadMetaFromDb, skipping DB restore');
+        return;
+      }
+      protocol = service;
+    } catch (e: unknown) {
+      ctx.logger.debug('Protocol service unavailable, skipping DB restore', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+      return;
+    }
+
+    // Phase 2: DB hydration (loads into SchemaRegistry)
+    try {
+      const { loaded, errors } = await protocol.loadMetaFromDb();
+
+      if (loaded > 0 || errors > 0) {
+        ctx.logger.info('Metadata restored from database to SchemaRegistry', { loaded, errors });
+      } else {
+        ctx.logger.debug('No persisted metadata found in database');
+      }
+    } catch (e: unknown) {
+      // Non-fatal: first-run or in-memory driver may not have sys_metadata yet
+      ctx.logger.debug('DB metadata restore failed (non-fatal)', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+    }
+  }
+
+  /**
+   * Bridge all SchemaRegistry objects to the metadata service.
+   *
+   * This ensures objects registered by plugins and loaded from sys_metadata
+   * are visible to AI tools and other consumers that query IMetadataService.
+   *
+   * Runs after both restoreMetadataFromDb() and syncRegisteredSchemas() to
+   * catch all objects in the SchemaRegistry regardless of their source.
+   */
+  private async bridgeObjectsToMetadataService(ctx: PluginContext): Promise<void> {
+    try {
+      const metadataService = ctx.getService<any>('metadata');
+      if (!metadataService || typeof metadataService.register !== 'function') {
+        ctx.logger.debug('Metadata service unavailable for bridging, skipping');
+        return;
+      }
+
+      if (!this.ql?.registry) {
+        ctx.logger.debug('SchemaRegistry unavailable for bridging, skipping');
+        return;
+      }
+
+      const objects = this.ql.registry.getAllObjects();
+      let bridged = 0;
+
+      for (const obj of objects) {
+        try {
+          // Check if object is already in metadata service to avoid duplicates
+          const existing = await metadataService.getObject(obj.name);
+          if (!existing) {
+            // Register object that exists in SchemaRegistry but not in metadata service
+            await metadataService.register('object', obj.name, obj);
+            bridged++;
+          }
+        } catch (e: unknown) {
+          ctx.logger.debug('Failed to bridge object to metadata service', {
+            object: obj.name,
+            error: e instanceof Error ? e.message : String(e),
+          });
+        }
+      }
+
+      if (bridged > 0) {
+        ctx.logger.info('Bridged objects from SchemaRegistry to metadata service', {
+          count: bridged,
+          total: objects.length
+        });
+      } else {
+        ctx.logger.debug('No objects needed bridging (all already in metadata service)');
+      }
+    } catch (e: unknown) {
+      ctx.logger.debug('Failed to bridge objects to metadata service', {
+        error: e instanceof Error ? e.message : String(e),
+      });
+    }
+  }
+
   /**
    * Load metadata from external metadata service into ObjectQL registry
    * This enables ObjectQL to use file-based or remote metadata

package/src/protocol.ts

@@ -2,15 +2,16 @@
 
 import { ObjectStackProtocol } from '@objectstack/spec/api';
 import { IDataEngine } from '@objectstack/core';
-import type { 
-    BatchUpdateRequest, 
-    BatchUpdateResponse, 
+import type {
+    BatchUpdateRequest,
+    BatchUpdateResponse,
     UpdateManyDataRequest,
     DeleteManyDataRequest
 } from '@objectstack/spec/api';
 import type { MetadataCacheRequest, MetadataCacheResponse, ServiceInfo, ApiRoutes, WellKnownCapabilities } from '@objectstack/spec/api';
 import type { IFeedService } from '@objectstack/spec/contracts';
 import { parseFilterAST, isFilterAST } from '@objectstack/spec/data';
+import { PLURAL_TO_SINGULAR, SINGULAR_TO_PLURAL } from '@objectstack/spec/shared';
 
 // We import SchemaRegistry directly since this class lives in the same package
 import { SchemaRegistry } from './registry.js';
@@ -180,24 +181,40 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
     }
 
     async getMetaTypes() {
-        return {
-            types: SchemaRegistry.getRegisteredTypes()
-        };
+        const schemaTypes = SchemaRegistry.getRegisteredTypes();
+
+        // Also include types from MetadataService (runtime-registered: agent, tool, etc.)
+        let runtimeTypes: string[] = [];
+        try {
+            const services = this.getServicesRegistry?.();
+            const metadataService = services?.get('metadata');
+            if (metadataService && typeof metadataService.getRegisteredTypes === 'function') {
+                runtimeTypes = await metadataService.getRegisteredTypes();
+            }
+        } catch {
+            // MetadataService not available
+        }
+
+        const allTypes = Array.from(new Set([...schemaTypes, ...runtimeTypes]));
+        return { types: allTypes };
     }
 
-    async getMetaItems(request: { type: string }) {
-        let items = SchemaRegistry.listItems(request.type);
-        // Normalize singular/plural: REST uses singular ('app') but registry may store as plural ('apps')
+    async getMetaItems(request: { type: string; packageId?: string }) {
+        const { packageId } = request;
+        let items = SchemaRegistry.listItems(request.type, packageId);
+        // Normalize singular/plural using explicit mapping
         if (items.length === 0) {
-            const alt = request.type.endsWith('s') ? request.type.slice(0, -1) : request.type + 's';
-            items = SchemaRegistry.listItems(alt);
+            const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+            if (alt) items = SchemaRegistry.listItems(alt, packageId);
         }
 
         // Fallback to database if registry is empty for this type
         if (items.length === 0) {
             try {
+                const whereClause: any = { type: request.type, state: 'active' };
+                if (packageId) whereClause._packageId = packageId;
                 const allRecords = await this.engine.find('sys_metadata', {
-                    where: { type: request.type, state: 'active' }
+                    where: whereClause
                 });
                 if (allRecords && allRecords.length > 0) {
                     items = allRecords.map((record: any) => {
@@ -209,8 +226,9 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                         return data;
                     });
                 } else {
-                    // Try alternate type name in DB
-                    const alt = request.type.endsWith('s') ? request.type.slice(0, -1) : request.type + 's';
+                    // Try alternate type name in DB using explicit mapping
+                    const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+                    if (alt) {
                     const altRecords = await this.engine.find('sys_metadata', {
                         where: { type: alt, state: 'active' }
                     });
@@ -223,24 +241,53 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                             return data;
                         });
                     }
+                    }
                 }
             } catch {
                 // DB not available, return registry results (empty)
             }
         }
 
+        // Merge with MetadataService (runtime-registered items: agents, tools, etc.)
+        try {
+            const services = this.getServicesRegistry?.();
+            const metadataService = services?.get('metadata');
+            if (metadataService && typeof metadataService.list === 'function') {
+                const runtimeItems = await metadataService.list(request.type);
+                if (runtimeItems && runtimeItems.length > 0) {
+                    // Merge, avoiding duplicates by name
+                    const itemMap = new Map<string, any>();
+                    for (const item of items) {
+                        const entry = item as any;
+                        if (entry && typeof entry === 'object' && 'name' in entry) {
+                            itemMap.set(entry.name, entry);
+                        }
+                    }
+                    for (const item of runtimeItems) {
+                        const entry = item as any;
+                        if (entry && typeof entry === 'object' && 'name' in entry) {
+                            itemMap.set(entry.name, entry);
+                        }
+                    }
+                    items = Array.from(itemMap.values());
+                }
+            }
+        } catch {
+            // MetadataService not available or doesn't support this type
+        }
+
         return {
             type: request.type,
             items
         };
     }
 
-    async getMetaItem(request: { type: string, name: string }) {
+    async getMetaItem(request: { type: string, name: string, packageId?: string }) {
         let item = SchemaRegistry.getItem(request.type, request.name);
-        // Normalize singular/plural
+        // Normalize singular/plural using explicit mapping
         if (item === undefined) {
-            const alt = request.type.endsWith('s') ? request.type.slice(0, -1) : request.type + 's';
-            item = SchemaRegistry.getItem(alt, request.name);
+            const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+            if (alt) item = SchemaRegistry.getItem(alt, request.name);
         }
 
         // Fallback to database if not in registry
@@ -256,8 +303,9 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                     // Hydrate back into registry for next time
                     SchemaRegistry.registerItem(request.type, item, 'name' as any);
                 } else {
-                    // Try alternate type name
-                    const alt = request.type.endsWith('s') ? request.type.slice(0, -1) : request.type + 's';
+                    // Try alternate type name using explicit mapping
+                    const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+                    if (alt) {
                     const altRecord = await this.engine.findOne('sys_metadata', {
                         where: { type: alt, name: request.name, state: 'active' }
                     });
@@ -268,12 +316,26 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                         // Hydrate back into registry for next time
                         SchemaRegistry.registerItem(request.type, item, 'name' as any);
                     }
+                    }
                 }
             } catch {
                 // DB not available, return undefined
             }
         }
 
+        // Fallback to MetadataService for runtime-registered items (agents, tools, etc.)
+        if (item === undefined) {
+            try {
+                const services = this.getServicesRegistry?.();
+                const metadataService = services?.get('metadata');
+                if (metadataService && typeof metadataService.get === 'function') {
+                    item = await metadataService.get(request.type, request.name);
+                }
+            } catch {
+                // MetadataService not available
+            }
+        }
+
         return {
             type: request.type,
             name: request.name,
@@ -481,10 +543,11 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
         }
         
         const records = await this.engine.find(request.object, options);
+        // Spec: FindDataResponseSchema — only `records` is returned.
+        // OData `value` adaptation (if needed) is handled in the HTTP dispatch layer.
         return {
             object: request.object,
-            value: records, // OData compatibility
-            records, // Legacy
+            records,
             total: records.length,
             hasMore: false
         };
@@ -559,7 +622,27 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
 
     async getMetaItemCached(request: { type: string, name: string, cacheRequest?: MetadataCacheRequest }): Promise<MetadataCacheResponse> {
         try {
-            const item = SchemaRegistry.getItem(request.type, request.name);
+            let item = SchemaRegistry.getItem(request.type, request.name);
+
+            // Normalize singular/plural using explicit mapping
+            if (!item) {
+                const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type];
+                if (alt) item = SchemaRegistry.getItem(alt, request.name);
+            }
+
+            // Fallback to MetadataService (e.g. agents, tools registered in MetadataManager)
+            if (!item) {
+                try {
+                    const services = this.getServicesRegistry?.();
+                    const metadataService = services?.get('metadata');
+                    if (metadataService && typeof metadataService.get === 'function') {
+                        item = await metadataService.get(request.type, request.name);
+                    }
+                } catch {
+                    // MetadataService not available
+                }
+            }
+
             if (!item) {
                 throw new Error(`Metadata item ${request.type}/${request.name} not found`);
             }
@@ -980,10 +1063,12 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
                     const data = typeof record.metadata === 'string'
                         ? JSON.parse(record.metadata)
                         : record.metadata;
-                    if (record.type === 'object') {
+                    // Normalize DB type to singular (DB may store legacy plural forms)
+                    const normalizedType = PLURAL_TO_SINGULAR[record.type] ?? record.type;
+                    if (normalizedType === 'object') {
                         SchemaRegistry.registerObject(data as any, record.packageId || 'sys_metadata');
                     } else {
-                        SchemaRegistry.registerItem(record.type, data, 'name' as any);
+                        SchemaRegistry.registerItem(normalizedType, data, 'name' as any);
                     }
                     loaded++;
                 } catch (e) {

package/src/registry.test.ts

@@ -53,11 +53,15 @@ describe('SchemaRegistry', () => {
             }).not.toThrow();
         });
 
-        it('should throw on namespace conflict', () => {
-            SchemaRegistry.registerNamespace('crm', 'com.example.crm');
-            expect(() => {
-                SchemaRegistry.registerNamespace('crm', 'com.other.crm');
-            }).toThrow(/already registered/);
+        it('should allow multiple packages to share a namespace', () => {
+            SchemaRegistry.registerNamespace('sys', 'com.objectstack.auth');
+            SchemaRegistry.registerNamespace('sys', 'com.objectstack.security');
+            // First registered package returned for backwards compat
+            expect(SchemaRegistry.getNamespaceOwner('sys')).toBe('com.objectstack.auth');
+            expect(SchemaRegistry.getNamespaceOwners('sys')).toEqual([
+                'com.objectstack.auth',
+                'com.objectstack.security',
+            ]);
         });
 
         it('should unregister namespace', () => {
@@ -65,6 +69,13 @@ describe('SchemaRegistry', () => {
             SchemaRegistry.unregisterNamespace('crm', 'com.example.crm');
             expect(SchemaRegistry.getNamespaceOwner('crm')).toBeUndefined();
         });
+
+        it('should keep namespace when one of multiple packages unregisters', () => {
+            SchemaRegistry.registerNamespace('sys', 'com.objectstack.auth');
+            SchemaRegistry.registerNamespace('sys', 'com.objectstack.setup');
+            SchemaRegistry.unregisterNamespace('sys', 'com.objectstack.setup');
+            expect(SchemaRegistry.getNamespaceOwner('sys')).toBe('com.objectstack.auth');
+        });
     });
 
     // ==========================================
@@ -342,17 +353,17 @@ describe('SchemaRegistry', () => {
     describe('Generic Metadata', () => {
         it('should register and retrieve generic items', () => {
             const item = { name: 'test_action', type: 'custom' };
-            SchemaRegistry.registerItem('actions', item, 'name', 'com.pkg');
-            
-            const retrieved = SchemaRegistry.getItem('actions', 'test_action');
+            SchemaRegistry.registerItem('action', item, 'name', 'com.pkg');
+
+            const retrieved = SchemaRegistry.getItem('action', 'test_action');
             expect(retrieved).toEqual(item);
         });
 
         it('should list items by type with package filter', () => {
-            SchemaRegistry.registerItem('actions', { name: 'a1' }, 'name', 'com.pkg1');
-            SchemaRegistry.registerItem('actions', { name: 'a2' }, 'name', 'com.pkg2');
-            
-            const filtered = SchemaRegistry.listItems('actions', 'com.pkg1');
+            SchemaRegistry.registerItem('action', { name: 'a1' }, 'name', 'com.pkg1');
+            SchemaRegistry.registerItem('action', { name: 'a2' }, 'name', 'com.pkg2');
+
+            const filtered = SchemaRegistry.listItems('action', 'com.pkg1');
             expect(filtered).toHaveLength(1);
         });
     });
@@ -385,12 +396,12 @@ describe('SchemaRegistry', () => {
     describe('Reset', () => {
         it('should clear all state', () => {
             SchemaRegistry.registerObject({ name: 'obj', fields: {} } as any, 'com.pkg', 'pkg', 'own');
-            SchemaRegistry.registerItem('actions', { name: 'act' }, 'name');
-            
+            SchemaRegistry.registerItem('action', { name: 'act' }, 'name');
+
             SchemaRegistry.reset();
-            
+
             expect(SchemaRegistry.getAllObjects()).toHaveLength(0);
-            expect(SchemaRegistry.listItems('actions')).toHaveLength(0);
+            expect(SchemaRegistry.listItems('action')).toHaveLength(0);
         });
     });