@objectstack/service-datasource 9.11.0 → 10.0.0

package/src/__tests__/datasource-admin-plugin.test.ts

@@ -229,3 +229,62 @@ describe('DatasourceAdminServicePlugin: persistence + bound count', () => {
     await expect(service.removeDatasource('reporting')).rejects.toThrow(/1 object\(s\)/);
   });
 });
+
+describe('DatasourceAdminServicePlugin: runtime datasource durability', () => {
+  /** In-memory `sys_metadata` engine shared across two boots (a "restart"). */
+  function fakeSysMetadataEngine() {
+    const rows: Array<Record<string, unknown>> = [];
+    return {
+      rows,
+      registerDriver() {},
+      registerDatasourceDef() {},
+      getDriverByName() { return undefined; },
+      findOne: async (_o: string, q: { where?: Record<string, unknown> }) => {
+        const w = q.where ?? {};
+        return rows.find((r) => Object.entries(w).every(([k, v]) => r[k] === v));
+      },
+      find: async (_o: string, q: { where?: Record<string, unknown> }) => {
+        const w = q.where ?? {};
+        return rows.filter((r) => Object.entries(w).every(([k, v]) => r[k] === v));
+      },
+      insert: async (_o: string, row: Record<string, unknown>) => { rows.push({ ...row }); },
+      update: async (_o: string, row: Record<string, unknown>, opts: { where: Record<string, unknown> }) => {
+        const i = rows.findIndex((r) => r.id === opts.where.id);
+        if (i >= 0) rows[i] = { ...rows[i], ...row };
+      },
+      delete: async (_o: string, opts: { where: Record<string, unknown> }) => {
+        const i = rows.findIndex((r) => r.id === opts.where.id);
+        if (i >= 0) rows.splice(i, 1);
+      },
+    };
+  }
+
+  it('persists a UI-created datasource to sys_metadata and restores it after a restart', async () => {
+    const data = fakeSysMetadataEngine();
+
+    // Boot #1: create a runtime sqlite datasource (no secret needed).
+    const b1 = await boot({ services: { data } });
+    await b1.service.createDatasource({ name: 'demo_ext', driver: 'sqlite', config: { filename: '/tmp/x.db' } });
+    // It is durably written to sys_metadata (not just the in-memory registry).
+    expect(data.rows.filter((r) => r.type === 'datasource' && r.name === 'demo_ext')).toHaveLength(1);
+
+    // Boot #2 = "restart": fresh in-memory registry, SAME sys_metadata engine.
+    const b2 = await boot({ services: { data } });
+    // Before restore, the fresh registry is empty.
+    expect(await b2.service.listDatasources()).toHaveLength(0);
+    // start() restores runtime rows from sys_metadata into the registry.
+    await b2.plugin.start(b2.ctx);
+    const after = await b2.service.listDatasources();
+    expect(after.map((d) => d.name)).toContain('demo_ext');
+    expect(after.find((d) => d.name === 'demo_ext')?.origin).toBe('runtime');
+  });
+
+  it('removes the durable sys_metadata row when a datasource is deleted', async () => {
+    const data = fakeSysMetadataEngine();
+    const b = await boot({ services: { data } });
+    await b.service.createDatasource({ name: 'gone', driver: 'sqlite', config: { filename: '/tmp/y.db' } });
+    expect(data.rows.some((r) => r.name === 'gone')).toBe(true);
+    await b.service.removeDatasource('gone');
+    expect(data.rows.some((r) => r.name === 'gone')).toBe(false);
+  });
+});

package/src/__tests__/datasource-admin-service.test.ts

@@ -286,3 +286,33 @@ describe('removeDatasource', () => {
     await expect(service.removeDatasource('nope')).rejects.toThrow(/not found/i);
   });
 });
+
+describe('getDatasource', () => {
+  it('returns config + hasSecret with the credentialsRef stripped', async () => {
+    const { service } = makeHarness({
+      seed: [
+        {
+          name: 'pg',
+          driver: 'postgres',
+          origin: 'runtime',
+          config: { host: 'db', port: 5432, database: 'app' },
+          external: { credentialsRef: 'sys_secret://datasource/pg#1' },
+        },
+      ],
+    });
+    const ds = await service.getDatasource('pg');
+    expect(ds).toMatchObject({ name: 'pg', driver: 'postgres', origin: 'runtime', hasSecret: true });
+    expect(ds!.config).toEqual({ host: 'db', port: 5432, database: 'app' });
+    // The opaque credential handle must never be returned.
+    expect(JSON.stringify(ds)).not.toContain('sys_secret');
+    expect(JSON.stringify(ds)).not.toContain('credentialsRef');
+  });
+
+  it('reports hasSecret:false and returns undefined for unknown names', async () => {
+    const { service } = makeHarness({
+      seed: [{ name: 'lite', driver: 'sqlite', origin: 'runtime', config: { filename: '/tmp/a.db' } }],
+    });
+    expect((await service.getDatasource('lite'))!.hasSecret).toBe(false);
+    expect(await service.getDatasource('missing')).toBeUndefined();
+  });
+});

package/src/admin-routes.ts

@@ -2,6 +2,7 @@
 
 import type { PluginContext } from '@objectstack/core';
 import type { IHttpServer } from '@objectstack/spec/contracts';
+import { DRIVER_CATALOG } from './driver-catalog.js';
 
 /**
  * Datasource lifecycle REST routes (ADR-0015 Addendum §3.5).
@@ -36,6 +37,14 @@ export function registerDatasourceAdminRoutes(
     }
   };
 
+  const externalService = (): any => {
+    try {
+      return ctx.getService<any>('external-datasource');
+    } catch {
+      return undefined;
+    }
+  };
+
   const unavailable = (res: any) =>
     res.status(503).json({ error: 'datasource_admin_unavailable' });
 
@@ -63,6 +72,72 @@ export function registerDatasourceAdminRoutes(
     res.json({ datasources });
   });
 
+  // Catalog of connection drivers + their JSON-Schema config (drives the
+  // Studio connection form). Static metadata — no service dependency, so it
+  // is always available even before any datasource-admin service is wired.
+  server.get(`${root}/drivers`, async (_req: any, res: any) => {
+    res.json({ drivers: DRIVER_CATALOG });
+  });
+
+  // Read-only schema introspection for the Studio "sync objects" flow.
+  // `GET /datasources/:name/remote-tables` lists the datasource's remote tables;
+  // `POST /datasources/:name/object-draft` generates an ObjectStack object
+  // definition draft for one table (introspect + type-map, no persistence —
+  // the caller creates the object through the normal metadata channel).
+  server.get(`${root}/:name/remote-tables`, async (req: any, res: any) => {
+    const svc = externalService();
+    if (!svc?.listRemoteTables) return unavailable(res);
+    try {
+      const tables = await svc.listRemoteTables(req.params.name);
+      res.json({ tables });
+    } catch (err) {
+      badRequest(res, err);
+    }
+  });
+
+  // Test a *saved* datasource by name with a live round-trip (backs the
+  // `datasource` `test_connection` action). Distinct from `POST /datasources/test`
+  // which probes an unsaved draft carried inline. Registered before the generic
+  // `:name` mutation routes.
+  // Read one datasource's full detail for the edit form (credential stripped;
+  // `config` is non-sensitive, plus a `hasSecret` flag). Registered after the
+  // static `/drivers` route so that literal segment is never captured as a name.
+  server.get(`${root}/:name`, async (req: any, res: any) => {
+    const svc = adminService();
+    if (!svc?.getDatasource) return unavailable(res);
+    try {
+      const datasource = await svc.getDatasource(req.params.name);
+      if (!datasource) return res.status(404).json({ error: 'not_found' });
+      res.json({ datasource });
+    } catch (err) {
+      badRequest(res, err);
+    }
+  });
+
+  server.post(`${root}/:name/test`, async (req: any, res: any) => {
+    const svc = externalService();
+    if (!svc?.testConnection) return unavailable(res);
+    try {
+      const result = await svc.testConnection(req.params.name);
+      res.json(result);
+    } catch (err) {
+      badRequest(res, err);
+    }
+  });
+
+  server.post(`${root}/:name/object-draft`, async (req: any, res: any) => {
+    const svc = externalService();
+    if (!svc?.generateObjectDraft) return unavailable(res);
+    const { table, ...opts } = (req.body as Record<string, unknown>) ?? {};
+    if (!table) return badRequest(res, new Error('Body field "table" is required.'));
+    try {
+      const draft = await svc.generateObjectDraft(req.params.name, String(table), opts);
+      res.json({ draft });
+    } catch (err) {
+      badRequest(res, err);
+    }
+  });
+
   // Probe a connection without persisting anything. Registered before the
   // `:name` routes so the literal `test` segment is never captured as a name.
   server.post(`${root}/test`, async (req: any, res: any) => {

package/src/datasource-admin-plugin.ts

@@ -33,6 +33,82 @@ interface DataEngineLike {
   registerDriver?: (driver: unknown, isDefault?: boolean) => void;
   registerDatasourceDef?: (def: { name: string; schemaMode?: string; external?: { allowWrites?: boolean } }) => void;
   getDriverByName?: (name: string) => unknown;
+  // sys_metadata CRUD used to persist runtime datasource records durably (same
+  // table runtime objects use). Optional — absent on lightweight kernels, in
+  // which case persistence degrades to in-memory (pre-existing behavior).
+  findOne?: (object: string, query: { where?: Record<string, unknown> }) => Promise<Record<string, unknown> | undefined | null>;
+  find?: (object: string, query: { where?: Record<string, unknown> }) => Promise<Record<string, unknown>[]>;
+  insert?: (object: string, row: Record<string, unknown>) => Promise<unknown>;
+  update?: (object: string, row: Record<string, unknown>, opts: { where: Record<string, unknown> }) => Promise<unknown>;
+  delete?: (object: string, opts: { where: Record<string, unknown> }) => Promise<unknown>;
+}
+
+/**
+ * Durable persistence for runtime datasource records via the `sys_metadata`
+ * table — the same store runtime objects use (the protocol writes objects there
+ * directly). `MetadataManager.register()` alone is in-memory unless a writable
+ * `datasource:` loader is wired, which standalone `serve` does not do; so a
+ * UI-created datasource vanished on restart. These helpers persist on write and
+ * the plugin restores them into the registry on boot before rehydrating pools.
+ * Credential cleartext is never stored — only the opaque `external.credentialsRef`.
+ */
+const DS_META_TYPE = 'datasource';
+const SYS_METADATA = 'sys_metadata';
+
+function newMetaId(): string {
+  return typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function'
+    ? crypto.randomUUID()
+    : `meta_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+}
+
+async function persistDatasourceRow(engine: DataEngineLike | undefined, record: { name: string }): Promise<void> {
+  if (!engine?.insert || !engine.findOne) return; // no durable store — in-memory only
+  const now = new Date().toISOString();
+  const existing = await engine.findOne(SYS_METADATA, {
+    where: { type: DS_META_TYPE, name: record.name, state: 'active' },
+  });
+  if (existing) {
+    await engine.update?.(
+      SYS_METADATA,
+      { metadata: JSON.stringify(record), updated_at: now, version: ((existing.version as number) || 0) + 1, state: 'active' },
+      { where: { id: existing.id } },
+    );
+  } else {
+    await engine.insert(SYS_METADATA, {
+      id: newMetaId(),
+      name: record.name,
+      type: DS_META_TYPE,
+      scope: 'platform',
+      metadata: JSON.stringify(record),
+      state: 'active',
+      version: 1,
+      created_at: now,
+      updated_at: now,
+    });
+  }
+}
+
+async function deleteDatasourceRow(engine: DataEngineLike | undefined, name: string): Promise<void> {
+  if (!engine?.findOne) return;
+  const existing = await engine.findOne(SYS_METADATA, { where: { type: DS_META_TYPE, name, state: 'active' } });
+  if (!existing) return;
+  if (engine.delete) await engine.delete(SYS_METADATA, { where: { id: existing.id } });
+  else await engine.update?.(SYS_METADATA, { state: 'inactive' }, { where: { id: existing.id } });
+}
+
+async function loadDatasourceRows(engine: DataEngineLike | undefined): Promise<Array<Record<string, unknown>>> {
+  if (!engine?.find) return [];
+  const rows = await engine.find(SYS_METADATA, { where: { type: DS_META_TYPE, state: 'active' } });
+  const out: Array<Record<string, unknown>> = [];
+  for (const r of rows ?? []) {
+    const raw = (r as { metadata?: unknown }).metadata;
+    try {
+      out.push(typeof raw === 'string' ? JSON.parse(raw) : (raw as Record<string, unknown>));
+    } catch {
+      /* skip corrupt row */
+    }
+  }
+  return out;
 }
 
 /**
@@ -147,7 +223,10 @@ export class DatasourceAdminServicePlugin implements Plugin {
         if (!metadata?.register) {
           throw new Error('Metadata service is unavailable; cannot persist datasource.');
         }
+        // In-memory registry (immediate visibility) + durable sys_metadata row
+        // (survives restart; restored on boot by restoreRuntimeDatasources).
         await metadata.register('datasource', record.name, record);
+        await persistDatasourceRow(engineOf(), record);
       },
 
       deleteDatasourceRecord: async (name) => {
@@ -156,6 +235,7 @@ export class DatasourceAdminServicePlugin implements Plugin {
           throw new Error('Metadata service is unavailable; cannot remove datasource.');
         }
         await metadata.unregister('datasource', name);
+        await deleteDatasourceRow(engineOf(), name);
       },
 
       writeSecret: async (input, hint) => {
@@ -223,16 +303,85 @@ export class DatasourceAdminServicePlugin implements Plugin {
     this.config = config;
     this.service = new DatasourceAdminService(config);
     ctx.registerService('datasource-admin', this.service);
+
+    // Setup-app nav (ADR-0029 D7): datasources are a *capability* this plugin
+    // owns, so it contributes its own entry into the `group_integrations` slot
+    // (core setup-nav must not fill capability-owned slots). datasource is a
+    // metadata type, so the entry opens the generic metadata-admin engine route
+    // rather than a bespoke page or an object view.
+    try {
+      const manifest = ctx.getService<{ register(m: any): void }>('manifest');
+      if (manifest && typeof manifest.register === 'function') {
+        manifest.register({
+          id: 'com.objectstack.service-datasource.nav',
+          namespace: 'sys',
+          version: this.version,
+          type: 'plugin',
+          scope: 'system',
+          name: 'Datasource Navigation',
+          description: 'Contributes the Datasources entry to the Setup app Integrations group.',
+          navigationContributions: [
+            {
+              app: 'setup',
+              group: 'group_integrations',
+              priority: 100,
+              items: [
+                {
+                  id: 'nav_datasources',
+                  type: 'url',
+                  label: 'Datasources',
+                  url: '/apps/setup/component/metadata/resource?type=datasource',
+                  icon: 'database',
+                  requiredPermissions: ['manage_platform_settings'],
+                },
+              ],
+            },
+          ],
+        });
+      }
+    } catch (err) {
+      this.options.logger?.warn?.('datasource nav contribution skipped', err);
+    }
   }
 
   async start(ctx: PluginContext): Promise<void> {
-    // Rebuild live connection pools for persisted runtime datasources before
-    // announcing readiness — a node restart otherwise leaves UI-created
-    // datasources with a record but no open pool until the next write.
+    // Restore UI-created (runtime) datasources from the durable sys_metadata
+    // store back into the in-memory registry, THEN rebuild their live pools.
+    // `register()` is in-memory only in standalone serve (no writable
+    // `datasource:` loader), so without this a node restart drops every
+    // UI-created datasource. Code-defined datasources come from the artifact and
+    // are unaffected.
+    await this.restoreRuntimeDatasources(ctx);
     await this.rehydratePools();
     if (this.service) await ctx.trigger('datasource-admin:ready', this.service);
   }
 
+  /** Reload persisted runtime datasource rows (sys_metadata) into the registry. */
+  private async restoreRuntimeDatasources(ctx: PluginContext): Promise<void> {
+    const engine = safeGetService<DataEngineLike>(ctx, 'data');
+    const metadata = safeGetService<MetadataServiceLike>(ctx, 'metadata');
+    if (!engine?.find || !metadata?.register) return;
+    let rows: Array<Record<string, unknown>>;
+    try {
+      rows = await loadDatasourceRows(engine);
+    } catch (err) {
+      this.options.logger?.warn?.('datasource restore: reading sys_metadata failed', err);
+      return;
+    }
+    let restored = 0;
+    for (const rec of rows) {
+      const name = (rec as { name?: string }).name;
+      if (!name) continue;
+      try {
+        await metadata.register('datasource', name, rec);
+        restored += 1;
+      } catch (err) {
+        this.options.logger?.warn?.(`datasource restore: register '${name}' failed`, err);
+      }
+    }
+    if (restored > 0) this.options.logger?.info?.(`datasource: restored ${restored} runtime record(s) from sys_metadata`);
+  }
+
   /**
    * Boot-time rehydration: list persisted runtime datasources and re-register
    * each one's connection pool (driver build → connect → registerDriver),

package/src/datasource-admin-service.ts

@@ -130,6 +130,36 @@ export class DatasourceAdminService implements IDatasourceAdminService {
     return summaries;
   }
 
+  /**
+   * Read one datasource's full detail for editing, with the credential stripped.
+   * Returns `config` (non-sensitive — credentials live in `sys_secret`, never in
+   * config), `origin`, and a `hasSecret` flag so the UI can show "leave blank to
+   * keep" without ever receiving the `credentialsRef` or any cleartext. Returns
+   * `undefined` when the name is unknown.
+   */
+  async getDatasource(name: string): Promise<
+    | (Pick<StoredDatasource, 'name' | 'label' | 'driver' | 'schemaMode' | 'config' | 'active' | 'definedIn'> & {
+        origin: 'code' | 'runtime';
+        hasSecret: boolean;
+      })
+    | undefined
+  > {
+    const rec = await this.config.getDatasourceRecord(name);
+    if (!rec) return undefined;
+    const hasSecret = Boolean(rec.external?.credentialsRef);
+    return {
+      name: rec.name,
+      label: rec.label,
+      driver: rec.driver,
+      schemaMode: rec.schemaMode ?? 'managed',
+      config: rec.config ?? {},
+      active: rec.active ?? true,
+      origin: rec.origin === 'runtime' ? 'runtime' : 'code',
+      hasSecret,
+      ...(rec.definedIn ? { definedIn: rec.definedIn } : {}),
+    };
+  }
+
   async testConnection(input: DatasourceDraft, secret?: SecretInput): Promise<TestConnectionResult> {
     if (!input?.driver) {
       return { ok: false, error: 'A driver is required to test a connection.' };

package/src/driver-catalog.ts

@@ -0,0 +1,113 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * Built-in datasource driver catalog.
+ *
+ * Each entry carries a JSON-Schema `configSchema` describing the driver's
+ * connection options, so the Studio UI can render a typed connection form
+ * instead of a raw-JSON editor (the `DriverDefinitionSchema.configSchema`
+ * contract — "Used by the UI to generate the connection form").
+ *
+ * Served by `GET /api/v1/datasources/drivers`. This is the curated set of
+ * connection drivers the connection form offers; a future runtime driver
+ * registry can supersede this list without changing the route contract.
+ */
+
+export interface DriverCatalogEntry {
+  /** Unique driver identifier used as `datasource.driver`. */
+  id: string;
+  /** Display label. */
+  label: string;
+  /** Optional one-line description. */
+  description?: string;
+  /** Optional Lucide icon name. */
+  icon?: string;
+  /** JSON Schema (draft-2020-12) for the driver's `config` object. */
+  configSchema: Record<string, unknown>;
+}
+
+const SSL_PROP = {
+  ssl: { type: 'boolean', title: 'Use SSL/TLS', default: false },
+} as const;
+
+export const DRIVER_CATALOG: DriverCatalogEntry[] = [
+  {
+    id: 'memory',
+    label: 'In-Memory',
+    description: 'Ephemeral in-memory driver for dev, tests, and prototyping. No connection settings.',
+    icon: 'memory-stick',
+    configSchema: { type: 'object', properties: {}, additionalProperties: false },
+  },
+  {
+    id: 'sqlite',
+    label: 'SQLite',
+    description: 'File-backed (or in-memory) SQL database. Great for local dev and small deployments.',
+    icon: 'database',
+    configSchema: {
+      type: 'object',
+      properties: {
+        filename: {
+          type: 'string',
+          title: 'Filename',
+          description: 'Database file path, or ":memory:" for an ephemeral in-memory database.',
+          default: ':memory:',
+        },
+      },
+      required: ['filename'],
+      additionalProperties: false,
+    },
+  },
+  {
+    id: 'postgres',
+    label: 'PostgreSQL',
+    description: 'PostgreSQL connection. Supply host/port/database or a connection URL.',
+    icon: 'database',
+    configSchema: {
+      type: 'object',
+      properties: {
+        url: { type: 'string', title: 'Connection URL', description: 'postgres://user:pass@host:5432/db (overrides the fields below when set).' },
+        host: { type: 'string', title: 'Host', default: 'localhost' },
+        port: { type: 'number', title: 'Port', default: 5432 },
+        database: { type: 'string', title: 'Database' },
+        username: { type: 'string', title: 'User' },
+        password: { type: 'string', title: 'Password', format: 'password' },
+        schema: { type: 'string', title: 'Schema', default: 'public' },
+        ...SSL_PROP,
+      },
+      additionalProperties: true,
+    },
+  },
+  {
+    id: 'mysql',
+    label: 'MySQL / MariaDB',
+    description: 'MySQL or MariaDB connection.',
+    icon: 'database',
+    configSchema: {
+      type: 'object',
+      properties: {
+        host: { type: 'string', title: 'Host', default: 'localhost' },
+        port: { type: 'number', title: 'Port', default: 3306 },
+        database: { type: 'string', title: 'Database' },
+        username: { type: 'string', title: 'User' },
+        password: { type: 'string', title: 'Password', format: 'password' },
+        ...SSL_PROP,
+      },
+      additionalProperties: true,
+    },
+  },
+  {
+    id: 'mongo',
+    label: 'MongoDB',
+    description: 'MongoDB connection via a connection URI.',
+    icon: 'database',
+    configSchema: {
+      type: 'object',
+      properties: {
+        url: { type: 'string', title: 'Connection URI', description: 'mongodb://host:27017' },
+        database: { type: 'string', title: 'Database' },
+      },
+      required: ['url'],
+      additionalProperties: true,
+    },
+  },
+];

package/src/external-datasource-service.ts

@@ -155,6 +155,31 @@ export class ExternalDatasourceService implements IExternalDatasourceService {
     return tables;
   }
 
+  /**
+   * Probe a *saved* datasource by name with a live round-trip. Reuses the
+   * introspection path (driver connect + schema read) as a cheap connectivity
+   * check, so the secret is resolved through the same wired pool as the rest of
+   * the introspection surface — the caller never handles cleartext. Returns a
+   * structured result rather than throwing so the route can render ok/error
+   * uniformly. This backs the `datasource` `test_connection` action
+   * (`POST /datasources/:name/test`).
+   */
+  async testConnection(
+    datasource: string,
+  ): Promise<{ ok: boolean; latencyMs?: number; tableCount?: number; error?: string }> {
+    const started = Date.now();
+    try {
+      const schema = await this.config.introspect(datasource);
+      return {
+        ok: true,
+        latencyMs: Date.now() - started,
+        tableCount: Object.keys(schema.tables).length,
+      };
+    } catch (err) {
+      return { ok: false, error: err instanceof Error ? err.message : String(err) };
+    }
+  }
+
   async generateObjectDraft(
     datasource: string,
     remoteName: string,