@objectstack/metadata 17.0.0-rc.1 → 17.0.0-rc.2

package/dist/node.d.cts

@@ -2,8 +2,8 @@ import { MetadataManager, MetadataManagerOptions, MetadataLoader, MetadataSerial
 export { DatabaseLoader, DatabaseLoaderOptions, HistoryCleanupManager, JSONSerializer, MemoryLoader, MetadataPlugin, Migration, RemoteLoader, SerializeOptions, TypeScriptSerializer, WatchCallback, YAMLSerializer, calculateChecksum, generateDiffSummary, generateSimpleDiff } from './index.cjs';
 export { HistoryOptions, MetaRef, MetadataEvent, MetadataItem, MetadataItemHeader, MetadataRepository, SysMetadataHistoryObject, SysMetadataObject, WatchFilter } from '@objectstack/metadata-core';
 import { MetadataLoaderContract, MetadataFormat, MetadataLoadOptions, MetadataLoadResult, MetadataStats, MetadataSaveOptions, MetadataSaveResult } from '@objectstack/spec/system';
-export { MetadataCollectionInfo, MetadataDiffResult, MetadataExportOptions, MetadataFormat, MetadataHistoryQueryOptions, MetadataHistoryQueryResult, MetadataHistoryRecord, MetadataHistoryRetentionPolicy, MetadataImportOptions, MetadataLoadOptions, MetadataLoadResult, MetadataLoaderContract, MetadataManagerConfig, MetadataSaveOptions, MetadataSaveResult, MetadataStats, MetadataWatchEvent } from '@objectstack/spec/system';
-export { IMetadataService, MetadataImportResult, MetadataTypeInfo, MetadataWatchCallback, MetadataWatchHandle } from '@objectstack/spec/contracts';
+export { MetadataCollectionInfo, MetadataDiffResult, MetadataFormat, MetadataHistoryQueryOptions, MetadataHistoryQueryResult, MetadataHistoryRecord, MetadataHistoryRetentionPolicy, MetadataLoadOptions, MetadataLoadResult, MetadataLoaderContract, MetadataManagerConfig, MetadataSaveOptions, MetadataSaveResult, MetadataStats, MetadataWatchEvent } from '@objectstack/spec/system';
+export { IMetadataService, MetadataExportOptions, MetadataImportOptions, MetadataImportResult, MetadataTypeInfo, MetadataWatchCallback, MetadataWatchHandle } from '@objectstack/spec/contracts';
 export { MetadataBulkResult, MetadataDependency, MetadataPluginConfig, MetadataPluginManifest, MetadataQuery, MetadataQueryResult, MetadataType, MetadataTypeRegistryEntry, MetadataValidationResult } from '@objectstack/spec/kernel';
 import { Logger } from '@objectstack/core';
 import 'zod';

package/dist/node.d.ts

@@ -2,8 +2,8 @@ import { MetadataManager, MetadataManagerOptions, MetadataLoader, MetadataSerial
 export { DatabaseLoader, DatabaseLoaderOptions, HistoryCleanupManager, JSONSerializer, MemoryLoader, MetadataPlugin, Migration, RemoteLoader, SerializeOptions, TypeScriptSerializer, WatchCallback, YAMLSerializer, calculateChecksum, generateDiffSummary, generateSimpleDiff } from './index.js';
 export { HistoryOptions, MetaRef, MetadataEvent, MetadataItem, MetadataItemHeader, MetadataRepository, SysMetadataHistoryObject, SysMetadataObject, WatchFilter } from '@objectstack/metadata-core';
 import { MetadataLoaderContract, MetadataFormat, MetadataLoadOptions, MetadataLoadResult, MetadataStats, MetadataSaveOptions, MetadataSaveResult } from '@objectstack/spec/system';
-export { MetadataCollectionInfo, MetadataDiffResult, MetadataExportOptions, MetadataFormat, MetadataHistoryQueryOptions, MetadataHistoryQueryResult, MetadataHistoryRecord, MetadataHistoryRetentionPolicy, MetadataImportOptions, MetadataLoadOptions, MetadataLoadResult, MetadataLoaderContract, MetadataManagerConfig, MetadataSaveOptions, MetadataSaveResult, MetadataStats, MetadataWatchEvent } from '@objectstack/spec/system';
-export { IMetadataService, MetadataImportResult, MetadataTypeInfo, MetadataWatchCallback, MetadataWatchHandle } from '@objectstack/spec/contracts';
+export { MetadataCollectionInfo, MetadataDiffResult, MetadataFormat, MetadataHistoryQueryOptions, MetadataHistoryQueryResult, MetadataHistoryRecord, MetadataHistoryRetentionPolicy, MetadataLoadOptions, MetadataLoadResult, MetadataLoaderContract, MetadataManagerConfig, MetadataSaveOptions, MetadataSaveResult, MetadataStats, MetadataWatchEvent } from '@objectstack/spec/system';
+export { IMetadataService, MetadataExportOptions, MetadataImportOptions, MetadataImportResult, MetadataTypeInfo, MetadataWatchCallback, MetadataWatchHandle } from '@objectstack/spec/contracts';
 export { MetadataBulkResult, MetadataDependency, MetadataPluginConfig, MetadataPluginManifest, MetadataQuery, MetadataQueryResult, MetadataType, MetadataTypeRegistryEntry, MetadataValidationResult } from '@objectstack/spec/kernel';
 import { Logger } from '@objectstack/core';
 import 'zod';

package/dist/node.js

@@ -169,6 +169,10 @@ var init_hmr_routes = __esm({
 
 // src/metadata-manager.ts
 import { getMetadataTypeActions } from "@objectstack/spec/kernel";
+import {
+  MetadataEventType,
+  MetadataEventSchema
+} from "@objectstack/spec/api";
 import { createLogger } from "@objectstack/core";
 
 // src/serializers/json-serializer.ts
@@ -507,6 +511,70 @@ var LRUCache = class {
   }
 };
 
+// src/utils/schema-sync-errors.ts
+var ALREADY_EXISTS = {
+  codes: /* @__PURE__ */ new Set([
+    // PostgreSQL SQLSTATE (class 42 — syntax error or access rule violation)
+    "42P07",
+    // duplicate_table
+    "42701",
+    // duplicate_column
+    "42710",
+    // duplicate_object — index / constraint already exists
+    // MySQL / MariaDB (mysql2 puts the symbolic name on `code`)
+    "ER_TABLE_EXISTS_ERROR",
+    // 1050
+    "ER_DUP_FIELDNAME",
+    // 1060
+    "ER_DUP_KEYNAME"
+    // 1061
+  ]),
+  errnos: /* @__PURE__ */ new Set([1050, 1060, 1061]),
+  /**
+   * Message fallback for drivers that carry no machine-readable code —
+   * notably SQLite, whose `code` is the undifferentiated `SQLITE_ERROR` for
+   * every DDL failure, so the message is the only signal available:
+   *   - `table sys_metadata already exists`
+   *   - `duplicate column name: environment_id`
+   *   - `index idx_x already exists`
+   * Postgres phrases its own as `relation "x" already exists` /
+   * `column "x" of relation "y" already exists`, which matches the same test.
+   */
+  message: /already exists|duplicate column name|duplicate key name/i
+};
+var MISSING_TABLE = {
+  codes: /* @__PURE__ */ new Set([
+    "42P01",
+    // PostgreSQL undefined_table
+    "ER_NO_SUCH_TABLE"
+    // MySQL / MariaDB 1146
+  ]),
+  errnos: /* @__PURE__ */ new Set([1146]),
+  /**
+   *   - SQLite / libsql: `no such table: sys_metadata_history`
+   *   - PostgreSQL:      `relation "sys_metadata_history" does not exist`
+   *   - MySQL/MariaDB:   `Table 'app.sys_metadata_history' doesn't exist`
+   */
+  message: /no such table|relation ["'`][^"'`]+["'`] does not exist|table ["'`][^"'`]+["'`] doesn'?t exist|unknown table/i
+};
+var MAX_CAUSE_DEPTH = 4;
+function matchesDriverError(error, signature, depth) {
+  if (error === null || error === void 0 || depth > MAX_CAUSE_DEPTH) return false;
+  if (typeof error === "string") return signature.message.test(error);
+  if (typeof error !== "object") return false;
+  const err = error;
+  if (typeof err.code === "string" && signature.codes.has(err.code)) return true;
+  if (typeof err.errno === "number" && signature.errnos.has(err.errno)) return true;
+  if (typeof err.message === "string" && signature.message.test(err.message)) return true;
+  return matchesDriverError(err.cause, signature, depth + 1);
+}
+function isSchemaAlreadyExistsError(error, depth = 0) {
+  return matchesDriverError(error, ALREADY_EXISTS, depth);
+}
+function isMissingTableError(error, depth = 0) {
+  return matchesDriverError(error, MISSING_TABLE, depth);
+}
+
 // src/migrations/add-sys-metadata-overlay-index.ts
 var INDEX_NAME = "idx_sys_metadata_overlay_active";
 var TABLE = "sys_metadata";
@@ -617,6 +685,19 @@ var DatabaseLoader = class {
     };
     this.schemaReady = false;
     this.historySchemaReady = false;
+    /**
+     * Whether the loud "DDL failed" report has already been printed for the
+     * metadata table / history table respectively. AGENTS.md → "Degradation log
+     * levels": say it **once**, at the first degradation, not once per retry.
+     */
+    this.schemaFailureReported = false;
+    this.historySchemaFailureReported = false;
+    /**
+     * Same once-only discipline for the #4825 seam: the history table is readable
+     * or it is not, and repeating the report per skipped write turns a real
+     * degradation into noise people learn to skim.
+     */
+    this.historySeqFailureReported = false;
     /**
      * Once-per-process dedupe for stored-row conversion notices — `load` /
      * `loadMany` are hot read paths (cached, but re-hit on every TTL expiry),
@@ -726,6 +807,24 @@ var DatabaseLoader = class {
    * Reads `MAX(event_seq) + 1` for the configured `organization_id`.
    * Legacy path — not transactional, so concurrent writes can collide.
    * The canonical (transactional) producer is `SysMetadataRepository`.
+   *
+   * #4825 (same shape as #4728, rule from #4632) — discriminate by error TYPE.
+   * This used to `catch { return 1 }`, with a comment that named BOTH reasons a
+   * read can fail and then answered both the same way. Exactly one of them is
+   * benign: the history table has not been provisioned, so there is no row to
+   * be inconsistent with and 1 genuinely IS the next number. Every other reason
+   * — connection drop, timeout, insufficient privileges — means the rows are
+   * still there and simply were not seen, and answering 1 against a table with
+   * N rows **collides with existing rows**: the insert succeeds, the log stays
+   * empty, and `event_seq` (the ordering key that history listing and rollback
+   * targeting both stand on) is silently wrong from then on. Note this is the
+   * costlier half of the #4728 family — not bytes that never landed, but bytes
+   * that landed *wrong*, which no retry and no restart repairs.
+   *
+   * @throws The underlying driver error, unchanged, for every non-benign read
+   *         failure. Deliberate: a sequence number this method cannot derive
+   *         from data it actually read is not a number it may invent. The
+   *         caller ({@link createHistoryRecord}) owns the consequence.
    */
   async nextEventSeq() {
     const where = this.organizationId ? { organization_id: this.organizationId } : {};
@@ -737,8 +836,9 @@ var DatabaseLoader = class {
         if (v > max) max = v;
       }
       return max + 1;
-    } catch {
-      return 1;
+    } catch (error) {
+      if (isMissingTableError(error)) return 1;
+      throw error;
     }
   }
   /**
@@ -775,17 +875,32 @@ var DatabaseLoader = class {
         ...SysMetadataObject,
         name: this.tableName
       });
-      this.schemaReady = true;
-      try {
-        await migrateProjectIdToEnvironmentId(this.driver);
-      } catch {
-      }
-      try {
-        await addSysMetadataOverlayIndex(this.driver);
-      } catch {
+    } catch (error) {
+      if (!isSchemaAlreadyExistsError(error)) {
+        if (!this.schemaFailureReported) {
+          this.schemaFailureReported = true;
+          console.error(
+            `[Metadata] DDL for the metadata table \`${this.tableName}\` FAILED \u2014 its table/columns were NOT created or altered. Every metadata write from here on (Studio saves, app installs, org overlays) targets storage that may not exist: writes will error out, or silently drop columns on a lenient driver, while the server keeps reporting healthy. This is NOT the benign "already exists" case \u2014 check the datasource/driver error below (insufficient privileges, datasource not connected, incompatible column type), fix it and restart. Schema sync is retried on the next metadata operation, so a transient cause recovers on its own.`,
+            error
+          );
+        }
+        return;
       }
+    }
+    if (this.schemaFailureReported) {
+      this.schemaFailureReported = false;
+      console.info(
+        `[Metadata] DDL for the metadata table \`${this.tableName}\` succeeded on retry \u2014 metadata writes are durable again.`
+      );
+    }
+    this.schemaReady = true;
+    try {
+      await migrateProjectIdToEnvironmentId(this.driver);
+    } catch {
+    }
+    try {
+      await addSysMetadataOverlayIndex(this.driver);
     } catch {
-      this.schemaReady = true;
     }
   }
   /**
@@ -803,9 +918,25 @@ var DatabaseLoader = class {
         ...SysMetadataHistoryObject,
         name: this.historyTableName
       });
+      if (this.historySchemaFailureReported) {
+        this.historySchemaFailureReported = false;
+        console.info(
+          `[Metadata] DDL for the metadata history table \`${this.historyTableName}\` succeeded on retry \u2014 change history is being recorded again.`
+        );
+      }
       this.historySchemaReady = true;
     } catch (error) {
-      console.error("Failed to ensure history schema, will retry on next operation:", error);
+      if (isSchemaAlreadyExistsError(error)) {
+        this.historySchemaReady = true;
+        return;
+      }
+      if (!this.historySchemaFailureReported) {
+        this.historySchemaFailureReported = true;
+        console.error(
+          `[Metadata] DDL for the metadata history table \`${this.historyTableName}\` FAILED \u2014 its table/columns were NOT created. Metadata change history (versions, diffs, rollback) will NOT be persisted while every metadata write keeps succeeding, so the audit trail silently ends here. Fix the datasource/driver error below and restart; the sync is retried on the next metadata operation.`,
+          error
+        );
+      }
     }
   }
   /**
@@ -846,7 +977,25 @@ var DatabaseLoader = class {
     }
     const historyId = generateId();
     const metadataJson = JSON.stringify(metadata);
-    const eventSeq = await this.nextEventSeq();
+    let eventSeq;
+    try {
+      eventSeq = await this.nextEventSeq();
+    } catch (error) {
+      if (!this.historySeqFailureReported) {
+        this.historySeqFailureReported = true;
+        console.error(
+          `[Metadata] Could not read \`${this.historyTableName}\` to determine the next \`event_seq\` \u2014 the history entry for ${type}/${name} was NOT written, and further entries are being skipped while this persists. The metadata write itself SUCCEEDED, so the server keeps looking healthy while its change history silently develops holes: version timelines and rollback targets will be incomplete. The entry is skipped deliberately \u2014 numbering it from 1 (what this code did before #4825) would collide with existing rows and make \`event_seq\` ordering wrong rather than merely incomplete, which nothing detects and no restart repairs. Fix the datasource/driver error below (connection, timeout, privileges); the next metadata write retries and reports recovery.`,
+          error
+        );
+      }
+      return;
+    }
+    if (this.historySeqFailureReported) {
+      this.historySeqFailureReported = false;
+      console.info(
+        `[Metadata] \`${this.historyTableName}\` is readable again \u2014 \`event_seq\` numbering recovered and change history is being recorded again. Entries skipped during the outage are not backfilled.`
+      );
+    }
     const historyRecord = {
       id: historyId,
       name,
@@ -1303,6 +1452,17 @@ function generateId() {
 }
 
 // src/metadata-manager.ts
+function generateEventUuid() {
+  const c = globalThis.crypto;
+  if (c && typeof c.randomUUID === "function") {
+    return c.randomUUID();
+  }
+  return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (ch) => {
+    const r = Math.random() * 16 | 0;
+    const v = ch === "x" ? r : r & 3 | 8;
+    return v.toString(16);
+  });
+}
 var _MetadataManager = class _MetadataManager {
   constructor(config) {
     this.loaders = /* @__PURE__ */ new Map();
@@ -1423,6 +1583,60 @@ var _MetadataManager = class _MetadataManager {
     this.realtimeService = service;
     this.logger.info("RealtimeService configured for metadata events");
   }
+  /**
+   * Publish a realtime {@link RealtimeMetadataEvent} for a metadata write
+   * (#4602 — contract-first).
+   *
+   * What reaches a `subscribeMetadata` callback must BE the spec's
+   * `MetadataEvent` (`@objectstack/spec/api`): `id` (uuid) at the top level,
+   * flattened `metadataType`/`name`/`definition`, `userId` when the write
+   * carried an actor. The transport keeps its `RealtimeEventPayload`
+   * envelope — `payload` carries the complete `MetadataEvent`, and the client
+   * SDK unwraps + validates it at the boundary.
+   *
+   * Two loud-by-design gates:
+   *  - `MetadataEventType` is a CLOSED enum. A metadata type outside it has
+   *    no declared realtime event contract, so we skip publishing (debug log)
+   *    instead of emitting an event every compliant consumer must reject.
+   *    Declared = enforced; widening coverage means widening the spec enum,
+   *    not producing off-contract events.
+   *  - The event body is `MetadataEventSchema.parse`d before publish, so a
+   *    malformed producer fails here (warn log, event not published) rather
+   *    than delivering a lie downstream.
+   */
+  async publishRealtimeMetadataEvent(action, type, name, opts = {}) {
+    if (!this.realtimeService) return;
+    const eventType = `metadata.${type}.${action}`;
+    if (!MetadataEventType.options.includes(eventType)) {
+      this.logger.debug(
+        `Metadata type '${type}' has no declared realtime event type (MetadataEventType) \u2014 skipping publish`,
+        { eventType, name }
+      );
+      return;
+    }
+    try {
+      const event = MetadataEventSchema.parse({
+        id: generateEventUuid(),
+        type: eventType,
+        metadataType: type,
+        name,
+        ...typeof opts.packageId === "string" ? { packageId: opts.packageId } : {},
+        ...opts.definition !== void 0 ? { definition: opts.definition } : {},
+        ...opts.userId ? { userId: opts.userId } : {},
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      });
+      const envelope = {
+        type: event.type,
+        object: type,
+        payload: { ...event },
+        timestamp: event.timestamp
+      };
+      await this.realtimeService.publish(envelope);
+      this.logger.debug(`Published ${eventType} event`, { name });
+    } catch (error) {
+      this.logger.warn(`Failed to publish metadata event`, { type, name, error });
+    }
+  }
   /**
    * Register a new metadata loader (data source)
    */
@@ -1466,25 +1680,11 @@ var _MetadataManager = class _MetadataManager {
         await loader.save(type, name, data);
       }
     }
-    if (this.realtimeService) {
-      const event = {
-        type: `metadata.${type}.created`,
-        object: type,
-        payload: {
-          metadataType: type,
-          name,
-          definition: data,
-          packageId: data?.packageId
-        },
-        timestamp: (/* @__PURE__ */ new Date()).toISOString()
-      };
-      try {
-        await this.realtimeService.publish(event);
-        this.logger.debug(`Published metadata.${type}.created event`, { name });
-      } catch (error) {
-        this.logger.warn(`Failed to publish metadata event`, { type, name, error });
-      }
-    }
+    await this.publishRealtimeMetadataEvent(existed ? "updated" : "created", type, name, {
+      definition: data,
+      packageId: data?.packageId,
+      userId: options?.userId
+    });
     if (options?.notify !== false) {
       this.notifyWatchers(type, {
         type: existed ? "changed" : "added",
@@ -1598,23 +1798,9 @@ var _MetadataManager = class _MetadataManager {
         }
       }
     }
-    if (this.realtimeService) {
-      const event = {
-        type: `metadata.${type}.deleted`,
-        object: type,
-        payload: {
-          metadataType: type,
-          name
-        },
-        timestamp: (/* @__PURE__ */ new Date()).toISOString()
-      };
-      try {
-        await this.realtimeService.publish(event);
-        this.logger.debug(`Published metadata.${type}.deleted event`, { name });
-      } catch (error) {
-        this.logger.warn(`Failed to publish metadata event`, { type, name, error });
-      }
-    }
+    await this.publishRealtimeMetadataEvent("deleted", type, name, {
+      userId: options?.userId
+    });
     if (options?.notify !== false) {
       this.notifyWatchers(type, {
         type: "deleted",
@@ -3315,6 +3501,14 @@ var MetadataPlugin = class {
      * initializes.
      */
     this.providesServices = ["metadata"];
+    /**
+     * init() registers the metadata system objects through the `manifest`
+     * service ObjectQLPlugin provides — order-if-present so that
+     * registration is deterministic instead of "whichever init ran first"
+     * (ADR-0116, #4471). Soft, not hard: without an engine the plugin
+     * degrades on purpose (objects are discovered via the legacy fallback).
+     */
+    this.optionalDependencies = ["com.objectstack.engine.objectql"];
     this.init = async (ctx) => {
       ctx.logger.info("Initializing Metadata Manager", {
         root: this.options.rootDir || process.cwd(),