@objectstack/objectql 4.0.2 → 4.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@objectstack/objectql",
-  "version": "4.0.2",
+  "version": "4.0.4",
   "license": "Apache-2.0",
   "description": "Isomorphic ObjectQL Engine for ObjectStack",
   "main": "dist/index.js",
@@ -13,13 +13,13 @@
     }
   },
   "dependencies": {
-    "@objectstack/core": "4.0.2",
-    "@objectstack/spec": "4.0.2",
-    "@objectstack/types": "4.0.2"
+    "@objectstack/core": "4.0.4",
+    "@objectstack/spec": "4.0.4",
+    "@objectstack/types": "4.0.4"
   },
   "devDependencies": {
     "typescript": "^6.0.2",
-    "vitest": "^4.1.2"
+    "vitest": "^4.1.4"
   },
   "scripts": {
     "build": "tsup --config ../../tsup.config.ts",

package/src/datasource-mapping.test.ts

@@ -0,0 +1,181 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect, beforeEach } from 'vitest';
+import { ObjectQL } from './engine.js';
+import { SchemaRegistry } from './registry.js';
+
+// Mock driver for testing
+const createMockDriver = (name: string) => ({
+  name,
+  version: '1.0.0',
+  supports: {},
+  connect: async () => {},
+  disconnect: async () => {},
+  checkHealth: async () => true,
+  find: async () => [],
+  findOne: async () => null,
+  create: async (obj: string, data: any) => ({ id: '1', ...data }),
+  update: async (obj: string, id: string, data: any) => ({ id, ...data }),
+  delete: async () => true,
+  count: async () => 0,
+  bulkCreate: async () => [],
+  bulkUpdate: async () => [],
+  bulkDelete: async () => {},
+  execute: async () => ({}),
+  findStream: async function* () {},
+  upsert: async (obj: string, data: any) => ({ id: '1', ...data }),
+  beginTransaction: async () => ({}),
+  commit: async () => {},
+  rollback: async () => {},
+  syncSchema: async () => {},
+});
+
+describe('DatasourceMapping', () => {
+  let engine: ObjectQL;
+
+  beforeEach(() => {
+    engine = new ObjectQL();
+    SchemaRegistry.reset();
+  });
+
+  it('should route objects by namespace', async () => {
+    const memoryDriver = createMockDriver('memory');
+    const tursoDriver = createMockDriver('turso');
+
+    engine.registerDriver(memoryDriver);
+    engine.registerDriver(tursoDriver, true); // default
+
+    // Configure mapping: crm namespace → memory
+    engine.setDatasourceMapping([
+      { namespace: 'crm', datasource: 'memory' },
+    ]);
+
+    // Register an object in crm namespace
+    SchemaRegistry.registerObject(
+      {
+        name: 'account',
+        fields: { name: { type: 'text' } },
+      },
+      'com.example.crm',
+      'crm',
+      'own'
+    );
+
+    // Test that it uses memory driver
+    const result = await engine.insert('account', { name: 'Test Account' });
+    expect(result).toBeDefined();
+    expect(result.name).toBe('Test Account');
+  });
+
+  it('should route objects by pattern', async () => {
+    const memoryDriver = createMockDriver('memory');
+    const tursoDriver = createMockDriver('turso');
+
+    engine.registerDriver(memoryDriver);
+    engine.registerDriver(tursoDriver, true);
+
+    // Configure mapping: sys_* pattern → turso
+    engine.setDatasourceMapping([
+      { objectPattern: 'sys_*', datasource: 'turso' },
+      { default: true, datasource: 'memory' },
+    ]);
+
+    // Register system objects
+    SchemaRegistry.registerObject(
+      {
+        name: 'sys_user',
+        fields: { username: { type: 'text' } },
+      },
+      'com.objectstack.system',
+      'system',
+      'own'
+    );
+
+    const result = await engine.insert('sys_user', { username: 'admin' });
+    expect(result).toBeDefined();
+  });
+
+  it('should respect priority order', async () => {
+    const memoryDriver = createMockDriver('memory');
+    const tursoDriver = createMockDriver('turso');
+
+    engine.registerDriver(memoryDriver);
+    engine.registerDriver(tursoDriver);
+
+    // Higher priority rule should win
+    engine.setDatasourceMapping([
+      { namespace: 'crm', datasource: 'memory', priority: 100 },
+      { namespace: 'crm', datasource: 'turso', priority: 50 }, // Lower number = higher priority
+    ]);
+
+    SchemaRegistry.registerObject(
+      {
+        name: 'account',
+        fields: { name: { type: 'text' } },
+      },
+      'com.example.crm',
+      'crm',
+      'own'
+    );
+
+    // Should use turso (priority 50) not memory (priority 100)
+    const result = await engine.insert('account', { name: 'Test' });
+    expect(result).toBeDefined();
+  });
+
+  it('should fallback to default rule', async () => {
+    const memoryDriver = createMockDriver('memory');
+    const tursoDriver = createMockDriver('turso');
+
+    engine.registerDriver(memoryDriver);
+    engine.registerDriver(tursoDriver);
+
+    engine.setDatasourceMapping([
+      { namespace: 'auth', datasource: 'turso' },
+      { default: true, datasource: 'memory' },
+    ]);
+
+    // Register object in different namespace
+    SchemaRegistry.registerObject(
+      {
+        name: 'task',
+        fields: { title: { type: 'text' } },
+      },
+      'com.example.todo',
+      'todo',
+      'own'
+    );
+
+    // Should use memory (default)
+    const result = await engine.insert('task', { title: 'Do something' });
+    expect(result).toBeDefined();
+  });
+
+  it('should prefer object explicit datasource over mapping', async () => {
+    const memoryDriver = createMockDriver('memory');
+    const tursoDriver = createMockDriver('turso');
+
+    engine.registerDriver(memoryDriver);
+    engine.registerDriver(tursoDriver);
+
+    engine.setDatasourceMapping([
+      { namespace: 'crm', datasource: 'memory' },
+    ]);
+
+    // Object explicitly sets datasource
+    SchemaRegistry.registerObject(
+      {
+        name: 'account',
+        datasource: 'turso', // Explicit override
+        fields: { name: { type: 'text' } },
+      },
+      'com.example.crm',
+      'crm',
+      'own'
+    );
+
+    // Should use turso (explicit) not memory (mapping)
+    const result = await engine.insert('account', { name: 'Test' });
+    expect(result).toBeDefined();
+  });
+});

package/src/engine.test.ts

@@ -6,6 +6,7 @@ import type { IDataDriver } from '@objectstack/spec/contracts';
 // Mock the SchemaRegistry to avoid side effects between tests
 vi.mock('./registry', () => {
   const mockObjects = new Map();
+  const mockContributors = new Map();
   return {
     SchemaRegistry: {
       getObject: vi.fn((name) => mockObjects.get(name)),
@@ -13,8 +14,18 @@ vi.mock('./registry', () => {
       registerObject: vi.fn((obj, packageId, namespace, ownership, priority) => {
         const fqn = namespace ? `${namespace}__${obj.name}` : obj.name;
         mockObjects.set(fqn, { ...obj, name: fqn });
+        // Also track contributors for getObjectOwner
+        if (!mockContributors.has(fqn)) {
+          mockContributors.set(fqn, []);
+        }
+        const contributors = mockContributors.get(fqn);
+        contributors.push({ packageId, namespace, ownership, priority, definition: obj });
         return fqn;
       }),
+      getObjectOwner: vi.fn((fqn) => {
+        const contributors = mockContributors.get(fqn);
+        return contributors?.find(c => c.ownership === 'own');
+      }),
       registerNamespace: vi.fn(),
       registerKind: vi.fn(),
       registerItem: vi.fn(),
@@ -25,7 +36,10 @@ vi.mock('./registry', () => {
         enabled: true,
         installedAt: new Date().toISOString(),
       })),
-      reset: vi.fn(() => mockObjects.clear()),
+      reset: vi.fn(() => {
+        mockObjects.clear();
+        mockContributors.clear();
+      }),
       metadata: {
         get: vi.fn(() => mockObjects) // Expose for verification if needed
       }

package/src/engine.ts

@@ -1,17 +1,19 @@
 // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
 
 import { QueryAST, HookContext, ServiceObject } from '@objectstack/spec/data';
-import { 
+import {
   EngineQueryOptions,
-  DataEngineInsertOptions, 
-  EngineUpdateOptions, 
+  DataEngineInsertOptions,
+  EngineUpdateOptions,
   EngineDeleteOptions,
   EngineAggregateOptions,
-  EngineCountOptions 
+  EngineCountOptions
 } from '@objectstack/spec/data';
 import { ExecutionContext, ExecutionContextSchema } from '@objectstack/spec/kernel';
 import { DriverInterface, IDataEngine, Logger, createLogger } from '@objectstack/core';
 import { CoreServiceName } from '@objectstack/spec/system';
+import { IRealtimeService, RealtimeEventPayload } from '@objectstack/spec/contracts';
+import { pluralToSingular } from '@objectstack/spec/shared';
 import { SchemaRegistry } from './registry.js';
 
 export type HookHandler = (context: HookContext) => Promise<void> | void;
@@ -69,7 +71,20 @@ export class ObjectQL implements IDataEngine {
   private drivers = new Map<string, DriverInterface>();
   private defaultDriver: string | null = null;
   private logger: Logger;
-  
+
+  // Datasource mapping rules (imported from defineStack)
+  private datasourceMapping: Array<{
+    namespace?: string;
+    package?: string;
+    objectPattern?: string;
+    default?: boolean;
+    datasource: string;
+    priority?: number;
+  }> = [];
+
+  // Package manifests registry (for defaultDatasource lookup)
+  private manifests = new Map<string, any>();
+
   // Per-object hooks with priority support
   private hooks: Map<string, HookEntry[]> = new Map([
     ['beforeFind', []], ['afterFind', []],
@@ -86,10 +101,13 @@ export class ObjectQL implements IDataEngine {
 
   // Action registry: key = "objectName:actionName"
   private actions = new Map<string, { handler: (ctx: any) => Promise<any> | any; package?: string }>();
-  
+
   // Host provided context additions (e.g. Server router)
   private hostContext: Record<string, any> = {};
 
+  // Realtime service for event publishing
+  private realtimeService?: IRealtimeService;
+
   constructor(hostContext: Record<string, any> = {}) {
     this.hostContext = hostContext;
     // Use provided logger or create a new one
@@ -303,6 +321,11 @@ export class ObjectQL implements IDataEngine {
       const namespace = manifest.namespace as string | undefined;
       this.logger.debug('Registering package manifest', { id, namespace });
 
+      // Store manifest for defaultDatasource lookup
+      if (id) {
+        this.manifests.set(id, manifest);
+      }
+
       // 1. Register the Package (manifest + lifecycle state)
       SchemaRegistry.installPackage(manifest);
       this.logger.debug('Installed Package', { id: manifest.id, name: manifest.name, namespace });
@@ -391,7 +414,7 @@ export class ObjectQL implements IDataEngine {
               for (const item of items) {
                   const itemName = item.name || item.id;
                   if (itemName) {
-                      SchemaRegistry.registerItem(key, item, 'name' as any, id);
+                      SchemaRegistry.registerItem(pluralToSingular(key), item, 'name' as any, id);
                   }
               }
           }
@@ -497,7 +520,7 @@ export class ObjectQL implements IDataEngine {
               for (const item of items) {
                   const itemName = item.name || item.id;
                   if (itemName) {
-                      SchemaRegistry.registerItem(key, item, 'name' as any, ownerId);
+                      SchemaRegistry.registerItem(pluralToSingular(key), item, 'name' as any, ownerId);
                   }
               }
           }
@@ -514,8 +537,8 @@ export class ObjectQL implements IDataEngine {
     }
 
     this.drivers.set(driver.name, driver);
-    this.logger.info('Registered driver', { 
-      driverName: driver.name, 
+    this.logger.info('Registered driver', {
+      driverName: driver.name,
       version: driver.version
     });
 
@@ -525,6 +548,17 @@ export class ObjectQL implements IDataEngine {
     }
   }
 
+  /**
+   * Set the realtime service for publishing data change events.
+   * Should be called after kernel resolves the realtime service.
+   *
+   * @param service - An IRealtimeService instance for event publishing
+   */
+  setRealtimeService(service: IRealtimeService): void {
+    this.realtimeService = service;
+    this.logger.info('RealtimeService configured for data events');
+  }
+
   /**
    * Helper to get object definition
    */
@@ -555,36 +589,138 @@ export class ObjectQL implements IDataEngine {
 
   /**
    * Helper to get the target driver
+   *
+   * Resolution priority (first match wins):
+   * 1. Object's explicit `datasource` field (if not 'default')
+   * 2. DatasourceMapping rules (namespace/package/pattern matching)
+   * 3. Package's `defaultDatasource` from manifest
+   * 4. Global default driver
    */
   private getDriver(objectName: string): DriverInterface {
     const object = SchemaRegistry.getObject(objectName);
-    
-    // 1. If object definition exists, check for explicit datasource
-    if (object) {
-      const datasourceName = object.datasource || 'default';
-      
-      // If configured for 'default', try to find the default driver
-      if (datasourceName === 'default') {
-        if (this.defaultDriver && this.drivers.has(this.defaultDriver)) {
-          return this.drivers.get(this.defaultDriver)!;
-        }
-      } else {
-        // Specific datasource requested
-        if (this.drivers.has(datasourceName)) {
-            return this.drivers.get(datasourceName)!;
+
+    // 1. Object's explicit datasource field (highest priority)
+    if (object?.datasource && object.datasource !== 'default') {
+      if (this.drivers.has(object.datasource)) {
+        return this.drivers.get(object.datasource)!;
+      }
+      throw new Error(`[ObjectQL] Datasource '${object.datasource}' configured for object '${objectName}' is not registered.`);
+    }
+
+    // 2. Check datasourceMapping rules
+    const mappedDatasource = this.resolveDatasourceFromMapping(objectName, object);
+    if (mappedDatasource && this.drivers.has(mappedDatasource)) {
+      this.logger.debug('Resolved datasource from mapping', {
+        object: objectName,
+        datasource: mappedDatasource
+      });
+      return this.drivers.get(mappedDatasource)!;
+    }
+
+    // 3. Check package's defaultDatasource
+    // Use the object's FQN name (from getObject) for ownership lookup
+    const fqn = object?.name || objectName;
+    const owner = SchemaRegistry.getObjectOwner(fqn);
+    if (owner?.packageId) {
+      const manifest = this.manifests.get(owner.packageId);
+      if (manifest?.defaultDatasource && manifest.defaultDatasource !== 'default') {
+        if (this.drivers.has(manifest.defaultDatasource)) {
+          this.logger.debug('Resolved datasource from package manifest', {
+            object: objectName,
+            package: owner.packageId,
+            datasource: manifest.defaultDatasource
+          });
+          return this.drivers.get(manifest.defaultDatasource)!;
         }
-        throw new Error(`[ObjectQL] Datasource '${datasourceName}' configured for object '${objectName}' is not registered.`);
       }
     }
 
-    // 2. Fallback for ad-hoc objects or missing definitions
-    if (this.defaultDriver) {
+    // 4. Fallback to global default driver
+    if (this.defaultDriver && this.drivers.has(this.defaultDriver)) {
       return this.drivers.get(this.defaultDriver)!;
     }
 
     throw new Error(`[ObjectQL] No driver available for object '${objectName}'`);
   }
 
+  /**
+   * Resolve datasource from mapping rules
+   *
+   * Rules are evaluated in order (or by priority if specified).
+   * First matching rule wins.
+   */
+  private resolveDatasourceFromMapping(
+    objectName: string,
+    object?: any
+  ): string | null {
+    if (!this.datasourceMapping || this.datasourceMapping.length === 0) {
+      return null;
+    }
+
+    // Sort rules by priority if any have priority set
+    const sortedRules = [...this.datasourceMapping].sort((a, b) => {
+      const aPriority = a.priority ?? 1000;
+      const bPriority = b.priority ?? 1000;
+      return aPriority - bPriority;
+    });
+
+    for (const rule of sortedRules) {
+      // 1. Match by namespace
+      if (rule.namespace && object?.namespace === rule.namespace) {
+        return rule.datasource;
+      }
+
+      // 2. Match by package ID
+      if (rule.package && object?.packageId === rule.package) {
+        return rule.datasource;
+      }
+
+      // 3. Match by object name pattern (glob-style)
+      if (rule.objectPattern && this.matchPattern(objectName, rule.objectPattern)) {
+        return rule.datasource;
+      }
+
+      // 4. Default fallback rule
+      if (rule.default) {
+        return rule.datasource;
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Simple glob pattern matching
+   * Supports * (any chars) and ? (single char)
+   */
+  private matchPattern(objectName: string, pattern: string): boolean {
+    const regexPattern = pattern
+      .replace(/[.+^${}()|[\]\\]/g, '\\$&')  // Escape regex special chars
+      .replace(/\*/g, '.*')                   // * → .*
+      .replace(/\?/g, '.');                   // ? → .
+
+    const regex = new RegExp(`^${regexPattern}$`);
+    return regex.test(objectName);
+  }
+
+  /**
+   * Set datasource mapping rules
+   * Called by ObjectQLPlugin during bootstrap
+   */
+  setDatasourceMapping(rules: Array<{
+    namespace?: string;
+    package?: string;
+    objectPattern?: string;
+    default?: boolean;
+    datasource: string;
+    priority?: number;
+  }>) {
+    this.datasourceMapping = rules;
+    this.logger.info('Datasource mapping rules configured', {
+      ruleCount: rules.length
+    });
+  }
+
   /**
    * Initialize the engine and all registered drivers
    */
@@ -594,14 +730,24 @@ export class ObjectQL implements IDataEngine {
       drivers: Array.from(this.drivers.keys())
     });
     
+    const failedDrivers: string[] = [];
     for (const [name, driver] of this.drivers) {
       try {
         await driver.connect();
         this.logger.info('Driver connected successfully', { driverName: name });
       } catch (e) {
+        failedDrivers.push(name);
         this.logger.error('Failed to connect driver', e as Error, { driverName: name });
       }
     }
+
+    if (failedDrivers.length > 0) {
+      this.logger.warn(
+        `${failedDrivers.length} of ${this.drivers.size} driver(s) failed initial connect. ` +
+        `Operations may recover via lazy reconnection or fail at query time.`,
+        { failedDrivers }
+      );
+    }
     
     this.logger.info('ObjectQL engine initialization complete');
   }
@@ -873,6 +1019,42 @@ export class ObjectQL implements IDataEngine {
         hookContext.result = result;
         await this.triggerHooks('afterInsert', hookContext);
 
+        // Publish data.record.created event to realtime service
+        if (this.realtimeService) {
+          try {
+            if (Array.isArray(result)) {
+              // Bulk insert - publish event for each record
+              for (const record of result) {
+                const event: RealtimeEventPayload = {
+                  type: 'data.record.created',
+                  object,
+                  payload: {
+                    recordId: record.id,
+                    after: record,
+                  },
+                  timestamp: new Date().toISOString(),
+                };
+                await this.realtimeService.publish(event);
+              }
+              this.logger.debug(`Published ${result.length} data.record.created events`, { object });
+            } else {
+              const event: RealtimeEventPayload = {
+                type: 'data.record.created',
+                object,
+                payload: {
+                  recordId: result.id,
+                  after: result,
+                },
+                timestamp: new Date().toISOString(),
+              };
+              await this.realtimeService.publish(event);
+              this.logger.debug('Published data.record.created event', { object, recordId: result.id });
+            }
+          } catch (error) {
+            this.logger.warn('Failed to publish data event', { object, error });
+          }
+        }
+
         return hookContext.result;
       } catch (e) {
         this.logger.error('Insert operation failed', e as Error, { object });
@@ -927,6 +1109,29 @@ export class ObjectQL implements IDataEngine {
            hookContext.event = 'afterUpdate';
            hookContext.result = result;
            await this.triggerHooks('afterUpdate', hookContext);
+
+           // Publish data.record.updated event to realtime service
+           if (this.realtimeService) {
+             try {
+               const resultId = (typeof result === 'object' && result && 'id' in result) ? (result as any).id : undefined;
+               const recordId = String(hookContext.input.id || resultId || '');
+               const event: RealtimeEventPayload = {
+                 type: 'data.record.updated',
+                 object,
+                 payload: {
+                   recordId,
+                   changes: hookContext.input.data,
+                   after: result,
+                 },
+                 timestamp: new Date().toISOString(),
+               };
+               await this.realtimeService.publish(event);
+               this.logger.debug('Published data.record.updated event', { object, recordId });
+             } catch (error) {
+               this.logger.warn('Failed to publish data event', { object, error });
+             }
+           }
+
            return hookContext.result;
        } catch (e) {
           this.logger.error('Update operation failed', e as Error, { object });
@@ -980,6 +1185,27 @@ export class ObjectQL implements IDataEngine {
           hookContext.event = 'afterDelete';
           hookContext.result = result;
           await this.triggerHooks('afterDelete', hookContext);
+
+          // Publish data.record.deleted event to realtime service
+          if (this.realtimeService) {
+            try {
+              const resultId = (typeof result === 'object' && result && 'id' in result) ? (result as any).id : undefined;
+              const recordId = String(hookContext.input.id || resultId || '');
+              const event: RealtimeEventPayload = {
+                type: 'data.record.deleted',
+                object,
+                payload: {
+                  recordId,
+                },
+                timestamp: new Date().toISOString(),
+              };
+              await this.realtimeService.publish(event);
+              this.logger.debug('Published data.record.deleted event', { object, recordId });
+            } catch (error) {
+              this.logger.warn('Failed to publish data event', { object, error });
+            }
+          }
+
           return hookContext.result;
       } catch (e) {
           this.logger.error('Delete operation failed', e as Error, { object });