@objectstack/service-datasource 9.11.0 → 10.2.0

package/src/datasource-admin-plugin.ts

@@ -4,7 +4,6 @@ import type { Plugin, PluginContext } from '@objectstack/core';
 import { registerMetadataTypeActions } from '@objectstack/spec/kernel';
 import type {
   IDatasourceDriverFactory,
-  DatasourceConnectionSpec,
   TestConnectionResult,
 } from './contracts/index.js';
 import {
@@ -13,6 +12,11 @@ import {
   type StoredDatasource,
   type ProbeInput,
 } from './datasource-admin-service.js';
+import {
+  DatasourceConnectionService,
+  type ConnectionEngineLike,
+} from './datasource-connection-service.js';
+import type { DatasourceConnectPolicy } from './contracts/connect-policy.js';
 import type { Logger } from './logger.js';
 
 /**
@@ -33,6 +37,82 @@ interface DataEngineLike {
   registerDriver?: (driver: unknown, isDefault?: boolean) => void;
   registerDatasourceDef?: (def: { name: string; schemaMode?: string; external?: { allowWrites?: boolean } }) => void;
   getDriverByName?: (name: string) => unknown;
+  // sys_metadata CRUD used to persist runtime datasource records durably (same
+  // table runtime objects use). Optional — absent on lightweight kernels, in
+  // which case persistence degrades to in-memory (pre-existing behavior).
+  findOne?: (object: string, query: { where?: Record<string, unknown> }) => Promise<Record<string, unknown> | undefined | null>;
+  find?: (object: string, query: { where?: Record<string, unknown> }) => Promise<Record<string, unknown>[]>;
+  insert?: (object: string, row: Record<string, unknown>) => Promise<unknown>;
+  update?: (object: string, row: Record<string, unknown>, opts: { where: Record<string, unknown> }) => Promise<unknown>;
+  delete?: (object: string, opts: { where: Record<string, unknown> }) => Promise<unknown>;
+}
+
+/**
+ * Durable persistence for runtime datasource records via the `sys_metadata`
+ * table — the same store runtime objects use (the protocol writes objects there
+ * directly). `MetadataManager.register()` alone is in-memory unless a writable
+ * `datasource:` loader is wired, which standalone `serve` does not do; so a
+ * UI-created datasource vanished on restart. These helpers persist on write and
+ * the plugin restores them into the registry on boot before rehydrating pools.
+ * Credential cleartext is never stored — only the opaque `external.credentialsRef`.
+ */
+const DS_META_TYPE = 'datasource';
+const SYS_METADATA = 'sys_metadata';
+
+function newMetaId(): string {
+  return typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function'
+    ? crypto.randomUUID()
+    : `meta_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+}
+
+async function persistDatasourceRow(engine: DataEngineLike | undefined, record: { name: string }): Promise<void> {
+  if (!engine?.insert || !engine.findOne) return; // no durable store — in-memory only
+  const now = new Date().toISOString();
+  const existing = await engine.findOne(SYS_METADATA, {
+    where: { type: DS_META_TYPE, name: record.name, state: 'active' },
+  });
+  if (existing) {
+    await engine.update?.(
+      SYS_METADATA,
+      { metadata: JSON.stringify(record), updated_at: now, version: ((existing.version as number) || 0) + 1, state: 'active' },
+      { where: { id: existing.id } },
+    );
+  } else {
+    await engine.insert(SYS_METADATA, {
+      id: newMetaId(),
+      name: record.name,
+      type: DS_META_TYPE,
+      scope: 'platform',
+      metadata: JSON.stringify(record),
+      state: 'active',
+      version: 1,
+      created_at: now,
+      updated_at: now,
+    });
+  }
+}
+
+async function deleteDatasourceRow(engine: DataEngineLike | undefined, name: string): Promise<void> {
+  if (!engine?.findOne) return;
+  const existing = await engine.findOne(SYS_METADATA, { where: { type: DS_META_TYPE, name, state: 'active' } });
+  if (!existing) return;
+  if (engine.delete) await engine.delete(SYS_METADATA, { where: { id: existing.id } });
+  else await engine.update?.(SYS_METADATA, { state: 'inactive' }, { where: { id: existing.id } });
+}
+
+async function loadDatasourceRows(engine: DataEngineLike | undefined): Promise<Array<Record<string, unknown>>> {
+  if (!engine?.find) return [];
+  const rows = await engine.find(SYS_METADATA, { where: { type: DS_META_TYPE, state: 'active' } });
+  const out: Array<Record<string, unknown>> = [];
+  for (const r of rows ?? []) {
+    const raw = (r as { metadata?: unknown }).metadata;
+    try {
+      out.push(typeof raw === 'string' ? JSON.parse(raw) : (raw as Record<string, unknown>));
+    } catch {
+      /* skip corrupt row */
+    }
+  }
+  return out;
 }
 
 /**
@@ -59,6 +139,13 @@ export interface DatasourceAdminServicePluginOptions {
   secrets?: SecretBinder;
   /** Override the driver factory (defaults to the `'datasource-driver-factory'` service). */
   driverFactory?: IDatasourceDriverFactory;
+  /**
+   * Host-injectable connect policy consulted before opening any datasource
+   * connection (ADR-0062 D5 / epic #2163 seam). Open-core default is permissive
+   * (allow); a multi-tenant host binds a stricter, fail-closed policy. Shared by
+   * both code-defined auto-connect and runtime-admin pool registration.
+   */
+  connectPolicy?: DatasourceConnectPolicy;
   logger?: Logger;
 }
 
@@ -86,6 +173,8 @@ export class DatasourceAdminServicePlugin implements Plugin {
 
   private service?: DatasourceAdminService;
   private config?: DatasourceAdminServiceConfig;
+  /** Shared "definition → live driver" path (ADR-0062 D1); also exposed as the `'datasource-connection'` service. */
+  private connection?: DatasourceConnectionService;
   private readonly options: DatasourceAdminServicePluginOptions;
 
   constructor(options: DatasourceAdminServicePluginOptions = {}) {
@@ -128,6 +217,19 @@ export class DatasourceAdminServicePlugin implements Plugin {
     const factory = (): IDatasourceDriverFactory | undefined =>
       this.options.driverFactory ?? safeGetService<IDatasourceDriverFactory>(ctx, 'datasource-driver-factory');
 
+    // The single "definition → live driver" path (ADR-0062 D1). Built here so
+    // the admin pool registration (runtime origin) and the app-plugin
+    // auto-connect (code origin) share one connect + lifecycle + policy path.
+    // Registered as a kernel service so `AppPlugin.start()` can resolve it.
+    this.connection = new DatasourceConnectionService({
+      factory,
+      engine: () => engineOf() as ConnectionEngineLike | undefined,
+      secrets: { resolve: (ref) => this.options.secrets?.resolve?.(ref) ?? Promise.resolve(undefined) },
+      policy: this.options.connectPolicy,
+      logger: this.options.logger,
+    });
+    ctx.registerService('datasource-connection', this.connection);
+
     const config: DatasourceAdminServiceConfig = {
       probe: (input) => this.probe(factory(), input),
 
@@ -147,7 +249,10 @@ export class DatasourceAdminServicePlugin implements Plugin {
         if (!metadata?.register) {
           throw new Error('Metadata service is unavailable; cannot persist datasource.');
         }
+        // In-memory registry (immediate visibility) + durable sys_metadata row
+        // (survives restart; restored on boot by restoreRuntimeDatasources).
         await metadata.register('datasource', record.name, record);
+        await persistDatasourceRow(engineOf(), record);
       },
 
       deleteDatasourceRecord: async (name) => {
@@ -156,6 +261,7 @@ export class DatasourceAdminServicePlugin implements Plugin {
           throw new Error('Metadata service is unavailable; cannot remove datasource.');
         }
         await metadata.unregister('datasource', name);
+        await deleteDatasourceRow(engineOf(), name);
       },
 
       writeSecret: async (input, hint) => {
@@ -181,40 +287,21 @@ export class DatasourceAdminServicePlugin implements Plugin {
         return objects.filter((o) => o?.datasource === datasource).length;
       },
 
+      // Hot pool (de)registration converges on the shared
+      // DatasourceConnectionService (ADR-0062 D1) — one connect path for code-
+      // and runtime-origin datasources. `connect()` builds the driver via the
+      // factory, dereferences `external.credentialsRef` through the SecretBinder,
+      // opens the connection, and registers the live driver + datasource def.
+      // Runtime-admin connects always degrade-with-warning on failure (never
+      // fail-fast), preserving the pre-ADR-0062 admin behavior.
       registerPool: async (record) => {
-        const f = factory();
-        const engine = engineOf();
-        if (!f || !engine?.registerDriver || !f.supports(record.driver)) return;
-        // Recover the cleartext credential from `sys_secret` so the pool opens
-        // with the real password. The cleartext is never persisted on the
-        // record (only `credentialsRef`), so it must be dereferenced here —
-        // both on create/update and on boot rehydration. Credential-less
-        // drivers (sqlite/memory) simply have no ref and skip this.
-        const credentialsRef = record.external?.credentialsRef;
-        const secret = credentialsRef ? await this.options.secrets?.resolve?.(credentialsRef) : undefined;
-        const handle = await f.create({ ...this.toSpec(record), ...(secret ? { secret } : {}) });
-        if (typeof handle?.connect === 'function') await handle.connect();
-        // The engine routes a datasource to a driver by `driver.name === <datasource name>`
-        // (see ObjectQL engine.getDriver). Prefer the factory's underlying engine
-        // driver (the `driver` escape hatch); fall back to the handle itself. Stamp
-        // the name so routing resolves to this pool.
-        const engineDriver = (handle.driver ?? handle) as { name?: string };
-        try {
-          engineDriver.name = record.name;
-        } catch {
-          /* frozen driver — registration may still work if name already matches */
-        }
-        engine.registerDriver(engineDriver);
-        engine.registerDatasourceDef?.({
-          name: record.name,
-          schemaMode: record.schemaMode,
-          external: record.external as { allowWrites?: boolean } | undefined,
+        await this.connection?.connect(record, {
+          context: { origin: record.origin ?? 'runtime', trigger: 'runtime-admin' },
         });
       },
 
       unregisterPool: async (name) => {
-        const driver = engineOf()?.getDriverByName?.(name) as { disconnect?: () => Promise<void> } | undefined;
-        if (typeof driver?.disconnect === 'function') await driver.disconnect();
+        await this.connection?.disconnect(name);
       },
 
       logger,
@@ -223,16 +310,85 @@ export class DatasourceAdminServicePlugin implements Plugin {
     this.config = config;
     this.service = new DatasourceAdminService(config);
     ctx.registerService('datasource-admin', this.service);
+
+    // Setup-app nav (ADR-0029 D7): datasources are a *capability* this plugin
+    // owns, so it contributes its own entry into the `group_integrations` slot
+    // (core setup-nav must not fill capability-owned slots). datasource is a
+    // metadata type, so the entry opens the generic metadata-admin engine route
+    // rather than a bespoke page or an object view.
+    try {
+      const manifest = ctx.getService<{ register(m: any): void }>('manifest');
+      if (manifest && typeof manifest.register === 'function') {
+        manifest.register({
+          id: 'com.objectstack.service-datasource.nav',
+          namespace: 'sys',
+          version: this.version,
+          type: 'plugin',
+          scope: 'system',
+          name: 'Datasource Navigation',
+          description: 'Contributes the Datasources entry to the Setup app Integrations group.',
+          navigationContributions: [
+            {
+              app: 'setup',
+              group: 'group_integrations',
+              priority: 100,
+              items: [
+                {
+                  id: 'nav_datasources',
+                  type: 'url',
+                  label: 'Datasources',
+                  url: '/apps/setup/component/metadata/resource?type=datasource',
+                  icon: 'database',
+                  requiredPermissions: ['manage_platform_settings'],
+                },
+              ],
+            },
+          ],
+        });
+      }
+    } catch (err) {
+      this.options.logger?.warn?.('datasource nav contribution skipped', err);
+    }
   }
 
   async start(ctx: PluginContext): Promise<void> {
-    // Rebuild live connection pools for persisted runtime datasources before
-    // announcing readiness — a node restart otherwise leaves UI-created
-    // datasources with a record but no open pool until the next write.
+    // Restore UI-created (runtime) datasources from the durable sys_metadata
+    // store back into the in-memory registry, THEN rebuild their live pools.
+    // `register()` is in-memory only in standalone serve (no writable
+    // `datasource:` loader), so without this a node restart drops every
+    // UI-created datasource. Code-defined datasources come from the artifact and
+    // are unaffected.
+    await this.restoreRuntimeDatasources(ctx);
     await this.rehydratePools();
     if (this.service) await ctx.trigger('datasource-admin:ready', this.service);
   }
 
+  /** Reload persisted runtime datasource rows (sys_metadata) into the registry. */
+  private async restoreRuntimeDatasources(ctx: PluginContext): Promise<void> {
+    const engine = safeGetService<DataEngineLike>(ctx, 'data');
+    const metadata = safeGetService<MetadataServiceLike>(ctx, 'metadata');
+    if (!engine?.find || !metadata?.register) return;
+    let rows: Array<Record<string, unknown>>;
+    try {
+      rows = await loadDatasourceRows(engine);
+    } catch (err) {
+      this.options.logger?.warn?.('datasource restore: reading sys_metadata failed', err);
+      return;
+    }
+    let restored = 0;
+    for (const rec of rows) {
+      const name = (rec as { name?: string }).name;
+      if (!name) continue;
+      try {
+        await metadata.register('datasource', name, rec);
+        restored += 1;
+      } catch (err) {
+        this.options.logger?.warn?.(`datasource restore: register '${name}' failed`, err);
+      }
+    }
+    if (restored > 0) this.options.logger?.info?.(`datasource: restored ${restored} runtime record(s) from sys_metadata`);
+  }
+
   /**
    * Boot-time rehydration: list persisted runtime datasources and re-register
    * each one's connection pool (driver build → connect → registerDriver),
@@ -277,16 +433,6 @@ export class DatasourceAdminServicePlugin implements Plugin {
 
   // --- internals -----------------------------------------------------------
 
-  private toSpec(record: StoredDatasource): DatasourceConnectionSpec {
-    return {
-      name: record.name,
-      driver: record.driver,
-      config: record.config ?? {},
-      external: record.external,
-      pool: record.pool,
-    };
-  }
-
   /** Probe a connection via the driver factory: build → connect → ping → close. */
   private async probe(
     factory: IDatasourceDriverFactory | undefined,

package/src/datasource-admin-service.ts

@@ -47,6 +47,8 @@ export interface StoredDatasource {
   external?: (Record<string, unknown> & { credentialsRef?: string }) | undefined;
   pool?: Record<string, unknown>;
   active?: boolean;
+  /** Force a live connection at boot even when managed + unrouted (ADR-0062 D2(c)). */
+  autoConnect?: boolean;
   origin?: 'code' | 'runtime';
   /** Package that defines a code-origin datasource, when known. */
   definedIn?: string;
@@ -130,6 +132,36 @@ export class DatasourceAdminService implements IDatasourceAdminService {
     return summaries;
   }
 
+  /**
+   * Read one datasource's full detail for editing, with the credential stripped.
+   * Returns `config` (non-sensitive — credentials live in `sys_secret`, never in
+   * config), `origin`, and a `hasSecret` flag so the UI can show "leave blank to
+   * keep" without ever receiving the `credentialsRef` or any cleartext. Returns
+   * `undefined` when the name is unknown.
+   */
+  async getDatasource(name: string): Promise<
+    | (Pick<StoredDatasource, 'name' | 'label' | 'driver' | 'schemaMode' | 'config' | 'active' | 'definedIn'> & {
+        origin: 'code' | 'runtime';
+        hasSecret: boolean;
+      })
+    | undefined
+  > {
+    const rec = await this.config.getDatasourceRecord(name);
+    if (!rec) return undefined;
+    const hasSecret = Boolean(rec.external?.credentialsRef);
+    return {
+      name: rec.name,
+      label: rec.label,
+      driver: rec.driver,
+      schemaMode: rec.schemaMode ?? 'managed',
+      config: rec.config ?? {},
+      active: rec.active ?? true,
+      origin: rec.origin === 'runtime' ? 'runtime' : 'code',
+      hasSecret,
+      ...(rec.definedIn ? { definedIn: rec.definedIn } : {}),
+    };
+  }
+
   async testConnection(input: DatasourceDraft, secret?: SecretInput): Promise<TestConnectionResult> {
     if (!input?.driver) {
       return { ok: false, error: 'A driver is required to test a connection.' };