@objectstack/runtime 4.0.1 → 4.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@objectstack/runtime",
-  "version": "4.0.1",
+  "version": "4.0.3",
   "license": "Apache-2.0",
   "description": "ObjectStack Core Runtime & Query Engine",
   "type": "module",
@@ -15,14 +15,14 @@
   },
   "dependencies": {
     "zod": "^4.3.6",
-    "@objectstack/core": "4.0.1",
-    "@objectstack/rest": "4.0.1",
-    "@objectstack/spec": "4.0.1",
-    "@objectstack/types": "4.0.1"
+    "@objectstack/core": "4.0.3",
+    "@objectstack/types": "4.0.3",
+    "@objectstack/spec": "4.0.3",
+    "@objectstack/rest": "4.0.3"
   },
   "devDependencies": {
     "typescript": "^6.0.2",
-    "vitest": "^4.1.2"
+    "vitest": "^4.1.4"
   },
   "scripts": {
     "build": "tsup --config ../../tsup.config.ts",

package/src/app-plugin.test.ts

@@ -49,10 +49,15 @@ describe('AppPlugin', () => {
             objects: []
         };
         const plugin = new AppPlugin(bundle);
-        
+
+        // Mock the manifest service
+        const mockManifestService = { register: vi.fn() };
+        vi.mocked(mockContext.getService).mockReturnValue(mockManifestService);
+
         await plugin.init(mockContext);
-        
-        expect(mockContext.registerService).toHaveBeenCalledWith('app.com.test.simple', bundle);
+
+        expect(mockContext.getService).toHaveBeenCalledWith('manifest');
+        expect(mockManifestService.register).toHaveBeenCalledWith(bundle);
     });
 
     it('start should do nothing if no runtime hooks', async () => {

package/src/app-plugin.ts

@@ -41,17 +41,13 @@ export class AppPlugin implements Plugin {
             version: this.version 
         });
         
-        // Register the app manifest as a service
-        // ObjectQLPlugin will discover this and call ql.registerApp()
-        const serviceName = `app.${appId}`;
-
-        // Merge manifest with the bundle to ensure objects/apps are accessible at root
-        // This supports both Legacy Manifests and new Stack Definitions
-        const servicePayload = this.bundle.manifest 
+        // Register the app manifest directly via the manifest service.
+        // This immediately decomposes the manifest into SchemaRegistry entries.
+        const servicePayload = this.bundle.manifest
             ? { ...this.bundle.manifest, ...this.bundle }
             : this.bundle;
 
-        ctx.registerService(serviceName, servicePayload);
+        ctx.getService<{ register(m: any): void }>('manifest').register(servicePayload);
     }
 
     start = async (ctx: PluginContext) => {

package/src/dispatcher-plugin.ts

@@ -12,7 +12,88 @@ export interface DispatcherPluginConfig {
 }
 
 /**
- * Send an HttpDispatcherResult through IHttpResponse
+ * Route definition emitted by service plugins (e.g. AIServicePlugin) via hooks.
+ * Minimal interface — matches the shape produced by `buildAIRoutes()`.
+ */
+interface RouteDefinition {
+    method: 'GET' | 'POST' | 'DELETE';
+    path: string;
+    description: string;
+    handler: (req: any) => Promise<any>;
+}
+
+/**
+ * Register a single RouteDefinition on the HTTP server.
+ * Returns true if the route was successfully registered.
+ */
+function mountRouteOnServer(route: RouteDefinition, server: IHttpServer, routePath: string): boolean {
+    const handler = async (req: any, res: any) => {
+        try {
+            const result = await route.handler({
+                body: req.body,
+                params: req.params,
+                query: req.query,
+            });
+
+            if (result.stream && result.events) {
+                // SSE streaming response
+                res.status(result.status);
+
+                // Apply headers from the route result if available
+                if (result.headers) {
+                    for (const [k, v] of Object.entries(result.headers)) {
+                        res.header(k, String(v));
+                    }
+                } else {
+                    res.header('Content-Type', 'text/event-stream');
+                    res.header('Cache-Control', 'no-cache');
+                    res.header('Connection', 'keep-alive');
+                }
+
+                // Write the stream — events are pre-encoded SSE strings
+                if (typeof res.write === 'function' && typeof res.end === 'function') {
+                    for await (const event of result.events) {
+                        res.write(typeof event === 'string' ? event : `data: ${JSON.stringify(event)}\n\n`);
+                    }
+                    res.end();
+                } else {
+                    // Fallback: collect events into array
+                    const events = [];
+                    for await (const event of result.events) {
+                        events.push(event);
+                    }
+                    res.json({ events });
+                }
+            } else {
+                res.status(result.status);
+                if (result.body !== undefined) {
+                    res.json(result.body);
+                } else {
+                    res.end();
+                }
+            }
+        } catch (err: any) {
+            errorResponse(err, res);
+        }
+    };
+
+    const m = route.method.toLowerCase();
+    if (m === 'get' && typeof server.get === 'function') {
+        server.get(routePath, handler);
+        return true;
+    } else if (m === 'post' && typeof server.post === 'function') {
+        server.post(routePath, handler);
+        return true;
+    } else if (m === 'delete' && typeof server.delete === 'function') {
+        server.delete(routePath, handler);
+        return true;
+    }
+    return false;
+}
+
+/**
+ * Send an HttpDispatcherResult through IHttpResponse.
+ * Differentiates between handled, unhandled (404), and special results.
  */
 function sendResult(result: HttpDispatcherResult, res: any): void {
     if (result.handled) {
@@ -32,7 +113,16 @@ function sendResult(result: HttpDispatcherResult, res: any): void {
             return;
         }
     }
-    res.status(404).json({ success: false, error: { message: 'Not Found', code: 404 } });
+    // Semantic 404: no route matched — include diagnostic info
+    res.status(404).json({
+        success: false,
+        error: {
+            message: 'Not Found',
+            code: 404,
+            type: 'ROUTE_NOT_FOUND',
+            hint: 'No handler matched this request. Check the API discovery endpoint for available routes.',
+        },
+    });
 }
 
 function errorResponse(err: any, res: any): void {
@@ -96,6 +186,16 @@ export function createDispatcherPlugin(config: DispatcherPluginConfig = {}): Plu
                 res.json({ data: await dispatcher.getDiscoveryInfo(prefix) });
             });
 
+            // ── Health ──────────────────────────────────────────────────
+            server.get(`${prefix}/health`, async (_req: any, res: any) => {
+                try {
+                    const result = await dispatcher.dispatch('GET', '/health', undefined, {}, { request: _req });
+                    sendResult(result, res);
+                } catch (err: any) {
+                    errorResponse(err, res);
+                }
+            });
+
             // ── Auth ────────────────────────────────────────────────────
             server.post(`${prefix}/auth/login`, async (req: any, res: any) => {
                 try {
@@ -359,6 +459,45 @@ export function createDispatcherPlugin(config: DispatcherPluginConfig = {}): Plu
             });
 
             ctx.logger.info('Dispatcher bridge routes registered', { prefix });
+
+            // ── Dynamic service routes (AI, etc.) ───────────────────
+            // Listen for route definitions emitted by service plugins.
+            // The AIServicePlugin emits 'ai:routes' with RouteDefinition[].
+            ctx.hook('ai:routes', async (routes: RouteDefinition[]) => {
+                if (!server) return;
+                for (const route of routes) {
+                    // Strip the /api/v1 prefix if present (it's already in the path)
+                    // and register on the HTTP server with the configured prefix.
+                    const routePath = route.path.startsWith('/api/v1')
+                        ? route.path
+                        : `${prefix}${route.path}`;
+                    mountRouteOnServer(route, server, routePath);
+                }
+                ctx.logger.info(`[Dispatcher] Registered ${routes.length} AI routes`);
+            });
+
+            // ── Fallback: recover routes cached before hook was registered ──
+            // If AIServicePlugin.start() ran before DispatcherPlugin.start()
+            // (possible when plugin start order differs from registration order),
+            // the 'ai:routes' trigger fires with no listener. The AIServicePlugin
+            // caches the routes on the kernel as __aiRoutes (see AIServicePlugin.start())
+            // as an internal cross-plugin protocol so we can recover them here.
+            // TODO: replace with a formal kernel.getCachedRoutes('ai') API in a future release.
+            const cachedRoutes = (kernel as any).__aiRoutes as RouteDefinition[] | undefined;
+            if (cachedRoutes && Array.isArray(cachedRoutes) && cachedRoutes.length > 0) {
+                let registered = 0;
+                for (const route of cachedRoutes) {
+                    const routePath = route.path.startsWith('/api/v1')
+                        ? route.path
+                        : `${prefix}${route.path}`;
+                    if (mountRouteOnServer(route, server, routePath)) {
+                        registered++;
+                    }
+                }
+                if (registered > 0) {
+                    ctx.logger.info(`[Dispatcher] Recovered ${registered} cached AI routes (hook timing fallback)`);
+                }
+            }
         },
     };
 }

package/src/http-dispatcher.root.test.ts

@@ -61,13 +61,16 @@ describe('HttpDispatcher Root Handling', () => {
         expect(data.routes).toBeDefined();
     });
 
-    it('should NOT handle POST request to root path ("")', async () => {
+    it('should return semantic 404 for POST request to root path ("")', async () => {
         const context = { request: {} };
         const method = 'POST';
         const path = '';
         
         const result = await dispatcher.dispatch(method, path, {}, {}, context);
 
-        expect(result.handled).toBe(false);
+        // The dispatcher now returns a typed 404 (ROUTE_NOT_FOUND) instead of { handled: false }
+        expect(result.handled).toBe(true);
+        expect(result.response?.status).toBe(404);
+        expect(result.response?.body?.error?.type).toBe('ROUTE_NOT_FOUND');
     });
 });

package/src/http-dispatcher.ts

@@ -2,6 +2,7 @@
 
 import { ObjectKernel, getEnv, resolveLocale } from '@objectstack/core';
 import { CoreServiceName } from '@objectstack/spec/system';
+import { pluralToSingular } from '@objectstack/spec/shared';
 
 /** Browser-safe UUID generator — prefers Web Crypto, falls back to RFC 4122 v4 */
 function randomUUID(): string {
@@ -59,6 +60,25 @@ export class HttpDispatcher {
         };
     }
 
+    /**
+     * 404 Route Not Found — no route is registered for this path.
+     */
+    private routeNotFound(route: string) {
+        return {
+            status: 404,
+            body: {
+                success: false,
+                error: {
+                    code: 404,
+                    message: `Route Not Found: ${route}`,
+                    type: 'ROUTE_NOT_FOUND' as const,
+                    route,
+                    hint: 'No route is registered for this path. Check the API discovery endpoint for available routes.',
+                },
+            },
+        };
+    }
+
     private ensureBroker() {
         if (!this.kernel.broker) {
             throw { statusCode: 500, message: 'Kernel Broker not available' };
@@ -133,11 +153,14 @@ export class HttpDispatcher {
         };
 
         // Build per-service status map
+        // handlerReady: true means the dispatcher has a real, bound handler for this route.
+        // handlerReady: false means the route is present in the discovery table but may not
+        // yet have a concrete implementation or may be served by a stub.
         const svcAvailable = (route?: string, provider?: string) => ({
-            enabled: true, status: 'available' as const, route, provider,
+            enabled: true, status: 'available' as const, handlerReady: true, route, provider,
         });
         const svcUnavailable = (name: string) => ({
-            enabled: false, status: 'unavailable' as const,
+            enabled: false, status: 'unavailable' as const, handlerReady: false,
             message: `Install a ${name} plugin to enable`,
         });
 
@@ -174,7 +197,7 @@ export class HttpDispatcher {
             },
             services: {
                 // Kernel-provided (always available via protocol implementation)
-                metadata:       { enabled: true, status: 'degraded' as const, route: routes.metadata, provider: 'kernel', message: 'In-memory registry; DB persistence pending' },
+                metadata:       { enabled: true, status: 'degraded' as const, handlerReady: true, route: routes.metadata, provider: 'kernel', message: 'In-memory registry; DB persistence pending' },
                 data:           svcAvailable(routes.data, 'kernel'),
                 // Plugin-provided — only available when a plugin registers the service
                 auth:           hasAuth ? svcAvailable(routes.auth) : svcUnavailable('auth'),
@@ -319,22 +342,82 @@ export class HttpDispatcher {
         
         // GET /metadata/types
         if (parts[0] === 'types') {
-            // Try protocol service for dynamic types
+            // PRIORITY 1: Try MetadataService directly (includes both typeRegistry with agent/tool AND runtime-registered types)
+            console.log('[HttpDispatcher] Attempting to resolve MetadataService...');
+            console.log('[HttpDispatcher] Available kernel methods:', {
+                hasGetServiceAsync: typeof this.kernel.getServiceAsync === 'function',
+                hasGetService: typeof this.kernel.getService === 'function',
+                hasContext: !!this.kernel.context,
+                hasContextGetService: typeof this.kernel.context?.getService === 'function',
+            });
+
+            // Try all service resolution paths with detailed logging
+            let metadataService: any = null;
+
+            // Path 1: kernel.getServiceAsync
+            if (typeof this.kernel.getServiceAsync === 'function') {
+                try {
+                    metadataService = await this.kernel.getServiceAsync('metadata');
+                    console.log('[HttpDispatcher] kernel.getServiceAsync("metadata") returned:', !!metadataService);
+                } catch (e: any) {
+                    console.log('[HttpDispatcher] kernel.getServiceAsync("metadata") failed:', e.message);
+                }
+            }
+
+            // Path 2: kernel.getService (if not found via async)
+            if (!metadataService && typeof this.kernel.getService === 'function') {
+                try {
+                    metadataService = await this.kernel.getService('metadata');
+                    console.log('[HttpDispatcher] kernel.getService("metadata") returned:', !!metadataService);
+                } catch (e: any) {
+                    console.log('[HttpDispatcher] kernel.getService("metadata") failed:', e.message);
+                }
+            }
+
+            // Path 3: kernel.context.getService (if not found)
+            if (!metadataService && this.kernel.context?.getService) {
+                try {
+                    metadataService = await this.kernel.context.getService('metadata');
+                    console.log('[HttpDispatcher] kernel.context.getService("metadata") returned:', !!metadataService);
+                } catch (e: any) {
+                    console.log('[HttpDispatcher] kernel.context.getService("metadata") failed:', e.message);
+                }
+            }
+
+            console.log('[HttpDispatcher] Final metadataService:', !!metadataService, 'has getRegisteredTypes:', typeof (metadataService as any)?.getRegisteredTypes);
+
+            if (metadataService && typeof (metadataService as any).getRegisteredTypes === 'function') {
+                try {
+                    const types = await (metadataService as any).getRegisteredTypes();
+                    console.log('[HttpDispatcher] MetadataService.getRegisteredTypes() returned:', types);
+                    return { handled: true, response: this.success({ types }) };
+                } catch (e: any) {
+                    // Log error but continue to fallbacks
+                    console.warn('[HttpDispatcher] MetadataService.getRegisteredTypes() failed:', e.message, e.stack);
+                }
+            } else {
+                console.log('[HttpDispatcher] MetadataService not available or missing getRegisteredTypes, falling back to protocol service');
+            }
+            // PRIORITY 2: Try protocol service (returns SchemaRegistry types only - missing agent/tool)
             const protocol = await this.resolveService('protocol');
             if (protocol && typeof protocol.getMetaTypes === 'function') {
                 const result = await protocol.getMetaTypes({});
+                console.log('[HttpDispatcher] Protocol service returned types:', result);
                 return { handled: true, response: this.success(result) };
             }
-            // Fallback: ask broker for registered types
+            // PRIORITY 3: ask broker for registered types
             if (broker) {
                 try {
                     const data = await broker.call('metadata.types', {}, { request: context.request });
+                    console.log('[HttpDispatcher] Broker returned types:', data);
                     return { handled: true, response: this.success(data) };
-                } catch {
+                } catch (e) {
+                    console.log('[HttpDispatcher] Broker call failed:', e);
                     // fall through to hardcoded defaults
                 }
             }
             // Last resort: hardcoded defaults
+            console.warn('[HttpDispatcher] Falling back to hardcoded defaults for metadata types');
             return { handled: true, response: this.success({ types: ['object', 'app', 'plugin'] }) };
         }
 
@@ -362,12 +445,14 @@ export class HttpDispatcher {
         // /metadata/:type/:name
         if (parts.length === 2) {
             const [type, name] = parts;
+            // Extract optional package filter from query string
+            const packageId = query?.package || undefined;
 
             // PUT /metadata/:type/:name (Save)
             if (method === 'PUT' && body) {
                 // Try to get the protocol service directly
                 const protocol = await this.resolveService('protocol');
-                
+
                 if (protocol && typeof protocol.saveMetaItem === 'function') {
                     try {
                         const result = await protocol.saveMetaItem({ type, name, item: body });
@@ -376,7 +461,7 @@ export class HttpDispatcher {
                         return { handled: true, response: this.error(e.message, 400) };
                     }
                 }
-                
+
                 // Fallback to broker if protocol not available (legacy)
                 if (broker) {
                     try {
@@ -405,15 +490,14 @@ export class HttpDispatcher {
                     return { handled: true, response: this.error('Not found', 404) };
                 }
 
-                // If type is singular (e.g. 'app'), use it directly
-                // If plural (e.g. 'apps'), slice it
-                const singularType = type.endsWith('s') ? type.slice(0, -1) : type;
-                
+                // Normalize plural URL paths to singular registry type names
+                const singularType = pluralToSingular(type);
+
                 // Try Protocol Service First (Preferred)
                 const protocol = await this.resolveService('protocol');
                 if (protocol && typeof protocol.getMetaItem === 'function') {
                      try {
-                        const data = await protocol.getMetaItem({ type: singularType, name });
+                        const data = await protocol.getMetaItem({ type: singularType, name, packageId });
                         return { handled: true, response: this.success(data) };
                      } catch (e: any) {
                         // Protocol might throw if not found or not supported
@@ -440,7 +524,7 @@ export class HttpDispatcher {
             const typeOrName = parts[0];
             // Extract optional package filter from query string
             const packageId = query?.package || undefined;
-            
+
             // Try protocol service first for any type
             const protocol = await this.resolveService('protocol');
             if (protocol && typeof protocol.getMetaItems === 'function') {
@@ -455,6 +539,22 @@ export class HttpDispatcher {
                 }
             }
 
+            // Try MetadataService directly for runtime-registered metadata (agents, tools, etc.)
+            const metadataService = await this.getService(CoreServiceName.enum.metadata);
+            if (metadataService && typeof (metadataService as any).list === 'function') {
+                try {
+                    const items = await (metadataService as any).list(typeOrName);
+                    if (items && items.length > 0) {
+                        return { handled: true, response: this.success({ type: typeOrName, items }) };
+                    }
+                } catch (e: any) {
+                    // MetadataService doesn't know this type or failed, continue to other fallbacks
+                    // Sanitize typeOrName to prevent log injection (CodeQL warning)
+                    const sanitizedType = String(typeOrName).replace(/[\r\n\t]/g, '');
+                    console.debug(`[HttpDispatcher] MetadataService.list() failed for type:`, sanitizedType, 'error:', e.message);
+                }
+            }
+
             // Try broker for the type
             if (broker) {
                 try {
@@ -1188,25 +1288,138 @@ export class HttpDispatcher {
         return s.charAt(0).toUpperCase() + s.slice(1);
     }
 
+    /**
+     * Handle AI service routes (/ai/chat, /ai/models, /ai/conversations, etc.)
+     * Resolves the AI service and its built-in route handlers, then dispatches.
+     */
+    async handleAI(subPath: string, method: string, body: any, query: any, _context: HttpProtocolContext): Promise<HttpDispatcherResult> {
+        let aiService: any;
+        try {
+            aiService = await this.resolveService('ai');
+        } catch {
+            // AI service not registered
+        }
+
+        if (!aiService) {
+            return {
+                handled: true,
+                response: {
+                    status: 404,
+                    body: { success: false, error: { message: 'AI service is not configured', code: 404 } },
+                },
+            };
+        }
+
+        // The AI service exposes route definitions via buildAIRoutes.
+        // We match the request path against known AI route patterns.
+        const fullPath = `/api/v1${subPath}`;
+
+        // Build a simple param-extracting matcher for route patterns like /api/v1/ai/conversations/:id
+        const matchRoute = (pattern: string, path: string): Record<string, string> | null => {
+            const patternParts = pattern.split('/');
+            const pathParts = path.split('/');
+            if (patternParts.length !== pathParts.length) return null;
+            const params: Record<string, string> = {};
+            for (let i = 0; i < patternParts.length; i++) {
+                if (patternParts[i].startsWith(':')) {
+                    params[patternParts[i].substring(1)] = pathParts[i];
+                } else if (patternParts[i] !== pathParts[i]) {
+                    return null;
+                }
+            }
+            return params;
+        };
+
+        // Try to get route definitions from the AI service's cached routes
+        const routes = (this.kernel as any).__aiRoutes as Array<{
+            method: string; path: string; handler: (req: any) => Promise<any>;
+        }> | undefined;
+
+        if (!routes) {
+            return {
+                handled: true,
+                response: {
+                    status: 503,
+                    body: { success: false, error: { message: 'AI service routes not yet initialized', code: 503 } },
+                },
+            };
+        }
+
+        for (const route of routes) {
+            if (route.method !== method) continue;
+            const params = matchRoute(route.path, fullPath);
+            if (params === null) continue;
+
+            const result = await route.handler({ body, params, query });
+
+            if (result.stream && result.events) {
+                // Return a streaming result for the adapter to handle
+                return {
+                    handled: true,
+                    result: {
+                        type: 'stream',
+                        contentType: result.vercelDataStream
+                            ? 'text/plain; charset=utf-8'
+                            : 'text/event-stream',
+                        events: result.events,
+                        vercelDataStream: result.vercelDataStream,
+                        headers: {
+                            'Content-Type': result.vercelDataStream
+                                ? 'text/plain; charset=utf-8'
+                                : 'text/event-stream',
+                            'Cache-Control': 'no-cache',
+                            'Connection': 'keep-alive',
+                        },
+                    },
+                };
+            }
+
+            return {
+                handled: true,
+                response: {
+                    status: result.status,
+                    body: result.body,
+                },
+            };
+        }
+
+        return {
+            handled: true,
+            response: this.routeNotFound(subPath),
+        };
+    }
+
     /**
      * Main Dispatcher Entry Point
      * Routes the request to the appropriate handler based on path and precedence
      */
-    async dispatch(method: string, path: string, body: any, query: any, context: HttpProtocolContext): Promise<HttpDispatcherResult> {
+    async dispatch(method: string, path: string, body: any, query: any, context: HttpProtocolContext, prefix?: string): Promise<HttpDispatcherResult> {
         const cleanPath = path.replace(/\/$/, ''); // Remove trailing slash if present, but strict on clean paths
 
         // 0. Discovery Endpoint (GET /discovery or GET /)
         // Standard route: /discovery (protocol-compliant)
         // Legacy route: / (empty path, for backward compatibility — MSW strips base URL)
         if ((cleanPath === '/discovery' || cleanPath === '') && method === 'GET') {
-             // We use '' as prefix since we are internal dispatcher
-             const info = await this.getDiscoveryInfo('');
+             const info = await this.getDiscoveryInfo(prefix ?? '');
              return { 
                  handled: true, 
                  response: this.success(info) 
              };
         }
 
+        // 0b. Health Endpoint (GET /health)
+        if (cleanPath === '/health' && method === 'GET') {
+            return {
+                handled: true,
+                response: this.success({
+                    status: 'ok',
+                    timestamp: new Date().toISOString(),
+                    version: '1.0.0',
+                    uptime: typeof process !== 'undefined' ? process.uptime() : undefined,
+                }),
+            };
+        }
+
         // 1. System Protocols (Prefix-based)
         if (cleanPath.startsWith('/auth')) {
             return this.handleAuth(cleanPath.substring(5), method, body, context);
@@ -1249,6 +1462,11 @@ export class HttpDispatcher {
              return this.handleI18n(cleanPath.substring(5), method, query, context);
         }
 
+        // AI Service — delegate to the registered AI route handlers
+        if (cleanPath.startsWith('/ai')) {
+             return this.handleAI(cleanPath, method, body, query, context);
+        }
+
         // OpenAPI Specification
         if (cleanPath === '/openapi.json' && method === 'GET') {
              const broker = this.ensureBroker();
@@ -1265,8 +1483,11 @@ export class HttpDispatcher {
         const result = await this.handleApiEndpoint(cleanPath, method, body, query, context);
         if (result.handled) return result;
 
-        // 3. Fallback (404)
-        return { handled: false };
+        // 3. Fallback — return semantic 404 with diagnostic info
+        return {
+            handled: true,
+            response: this.routeNotFound(cleanPath),
+        };
     }
 
     /**