@objectstack/metadata 4.2.0 → 5.1.0

package/dist/index.cjs

@@ -64,7 +64,12 @@ function registerMetadataHmrRoutes(app, manager, options = {}) {
           metadataType: evt.metadataType ?? type,
           name: evt.name ?? "",
           path: evt.path,
-          timestamp: Number.isFinite(ts) ? ts : Date.now()
+          timestamp: Number.isFinite(ts) ? ts : Date.now(),
+          // Forward the canonical server-side sequence number when the
+          // event originated from a MetadataRepository (ADR-0008). Legacy
+          // chokidar-driven events have no seq — clients fall back to
+          // their local counter in that case.
+          ...typeof evt.seq === "number" ? { seq: evt.seq } : {}
         });
       });
     }
@@ -603,7 +608,7 @@ var DatabaseLoader = class {
     this.tableName = options.tableName ?? "sys_metadata";
     this.historyTableName = options.historyTableName ?? "sys_metadata_history";
     this.organizationId = options.organizationId;
-    this.projectId = options.projectId;
+    void options.projectId;
     this.trackHistory = options.trackHistory !== false;
     const cacheOpts = options.cache;
     const cacheEnabled = cacheOpts?.enabled !== false;
@@ -693,6 +698,26 @@ var DatabaseLoader = class {
     }
     return this.driver.delete(table, id);
   }
+  /**
+   * Compute the next per-org `event_seq` for `sys_metadata_history`.
+   * Reads `MAX(event_seq) + 1` for the configured `organization_id`.
+   * Legacy path — not transactional, so concurrent writes can collide.
+   * The canonical (transactional) producer is `SysMetadataRepository`.
+   */
+  async nextEventSeq() {
+    const where = this.organizationId ? { organization_id: this.organizationId } : {};
+    try {
+      const rows = await this._find(this.historyTableName, { where });
+      let max = 0;
+      for (const row of rows) {
+        const v = typeof row.event_seq === "number" ? row.event_seq : 0;
+        if (v > max) max = v;
+      }
+      return max + 1;
+    } catch {
+      return 1;
+    }
+  }
   /**
    * Ensure the metadata table exists.
    * Uses IDataDriver.syncSchema with the SysMetadataObject definition
@@ -757,8 +782,9 @@ var DatabaseLoader = class {
   }
   /**
    * Build base filter conditions for queries.
-   * Filters by organizationId when configured; project_id when projectId is set,
-   * or null (platform-global) when not set.
+   * Filters by organizationId when configured. `projectId` is accepted
+   * for back-compat but no longer constrains the query — see
+   * ADR-0008 §0 (branch/project removal).
    */
   baseFilter(type, name) {
     const filter = { type };
@@ -768,13 +794,11 @@ var DatabaseLoader = class {
     if (this.organizationId) {
       filter.organization_id = this.organizationId;
     }
-    filter.project_id = this.projectId ?? null;
     return filter;
   }
   /**
    * Create a history record for a metadata change.
    *
-   * @param metadataId - The metadata record ID
    * @param type - Metadata type
    * @param name - Metadata name
    * @param version - Version number
@@ -784,7 +808,7 @@ var DatabaseLoader = class {
    * @param changeNote - Optional change description
    * @param recordedBy - Optional user who made the change
    */
-  async createHistoryRecord(metadataId, type, name, version, metadata, operationType, previousChecksum, changeNote, recordedBy) {
+  async createHistoryRecord(type, name, version, metadata, operationType, previousChecksum, changeNote, recordedBy) {
     if (!this.trackHistory) return;
     await this.ensureHistorySchema();
     const now = (/* @__PURE__ */ new Date()).toISOString();
@@ -794,9 +818,9 @@ var DatabaseLoader = class {
     }
     const historyId = generateId();
     const metadataJson = JSON.stringify(metadata);
+    const eventSeq = await this.nextEventSeq();
     const historyRecord = {
       id: historyId,
-      metadataId,
       name,
       type,
       version,
@@ -807,13 +831,12 @@ var DatabaseLoader = class {
       changeNote,
       recordedBy,
       recordedAt: now,
-      ...this.organizationId ? { organizationId: this.organizationId } : {},
-      ...this.projectId !== void 0 ? { projectId: this.projectId } : {}
+      ...this.organizationId ? { organizationId: this.organizationId } : {}
     };
     try {
       await this._create(this.historyTableName, {
         id: historyRecord.id,
-        metadata_id: historyRecord.metadataId,
+        event_seq: eventSeq,
         name: historyRecord.name,
         type: historyRecord.type,
         version: historyRecord.version,
@@ -824,8 +847,8 @@ var DatabaseLoader = class {
         change_note: historyRecord.changeNote,
         recorded_by: historyRecord.recordedBy,
         recorded_at: historyRecord.recordedAt,
-        ...this.organizationId ? { organization_id: this.organizationId } : {},
-        ...this.projectId !== void 0 ? { project_id: this.projectId } : {}
+        source: "database-loader",
+        ...this.organizationId ? { organization_id: this.organizationId } : {}
       });
     } catch (error) {
       console.error(`Failed to create history record for ${type}/${name}:`, error);
@@ -1001,25 +1024,20 @@ var DatabaseLoader = class {
   async getHistoryRecord(type, name, version) {
     if (!this.trackHistory) return null;
     await this.ensureHistorySchema();
-    const metadataRow = await this._findOne(this.tableName, {
-      where: this.baseFilter(type, name)
-    });
-    if (!metadataRow) return null;
     const filter = {
-      metadata_id: metadataRow.id,
+      type,
+      name,
       version
     };
     if (this.organizationId) {
       filter.organization_id = this.organizationId;
     }
-    filter.project_id = this.projectId ?? null;
     const row = await this._findOne(this.historyTableName, {
       where: filter
     });
     if (!row) return null;
     return {
       id: row.id,
-      metadataId: row.metadata_id,
       name: row.name,
       type: row.type,
       version: row.version,
@@ -1029,7 +1047,6 @@ var DatabaseLoader = class {
       previousChecksum: row.previous_checksum,
       changeNote: row.change_note,
       organizationId: row.organization_id,
-      projectId: row.project_id,
       recordedBy: row.recorded_by,
       recordedAt: row.recorded_at
     };
@@ -1045,18 +1062,11 @@ var DatabaseLoader = class {
     }
     await this.ensureSchema();
     await this.ensureHistorySchema();
-    const filter = { type, name };
-    if (this.organizationId) filter.organization_id = this.organizationId;
-    filter.project_id = this.projectId ?? null;
-    const metadataRecord = await this._findOne(this.tableName, { where: filter });
-    if (!metadataRecord) {
-      return { records: [], total: 0, hasMore: false };
-    }
     const historyFilter = {
-      metadata_id: metadataRecord.id
+      type,
+      name
     };
     if (this.organizationId) historyFilter.organization_id = this.organizationId;
-    historyFilter.project_id = this.projectId ?? null;
     if (options?.operationType) historyFilter.operation_type = options.operationType;
     if (options?.since) historyFilter.recorded_at = { $gte: options.since };
     if (options?.until) {
@@ -1085,7 +1095,6 @@ var DatabaseLoader = class {
       const parsedMetadata = typeof row.metadata === "string" ? JSON.parse(row.metadata) : row.metadata;
       return {
         id: row.id,
-        metadataId: row.metadata_id,
         name: row.name,
         type: row.type,
         version: row.version,
@@ -1095,7 +1104,6 @@ var DatabaseLoader = class {
         previousChecksum: row.previous_checksum,
         changeNote: row.change_note,
         organizationId: row.organization_id,
-        projectId: row.project_id,
         recordedBy: row.recorded_by,
         recordedAt: row.recorded_at
       };
@@ -1131,7 +1139,6 @@ var DatabaseLoader = class {
     });
     this.invalidate(type, name);
     await this.createHistoryRecord(
-      existing.id,
       type,
       name,
       newVersion,
@@ -1173,7 +1180,6 @@ var DatabaseLoader = class {
         });
         this.invalidate(type, name);
         await this.createHistoryRecord(
-          existing.id,
           type,
           name,
           version,
@@ -1202,13 +1208,11 @@ var DatabaseLoader = class {
           version: 1,
           source: "database",
           ...this.organizationId ? { organization_id: this.organizationId } : {},
-          ...this.projectId !== void 0 ? { project_id: this.projectId } : { project_id: null },
           created_at: now,
           updated_at: now
         });
         this.invalidate(type, name);
         await this.createHistoryRecord(
-          id,
           type,
           name,
           1,
@@ -1275,6 +1279,7 @@ var _MetadataManager = class _MetadataManager {
     // Invalidated on every `register()` / `unregister()` to keep CRUD writes
     // visible to subsequent reads.
     this.listCache = /* @__PURE__ */ new Map();
+    this.repoWatchClosed = false;
     this.config = config;
     this.logger = (0, import_core.createLogger)({ level: "info", format: "pretty" });
     this.serializers = /* @__PURE__ */ new Map();
@@ -2282,6 +2287,110 @@ var _MetadataManager = class _MetadataManager {
    */
   async stopWatching() {
   }
+  // ─── ADR-0008 PR-6: Repository wiring ───────────────────────────────
+  /**
+   * Attach a {@link MetadataRepository} as a supplementary event source.
+   *
+   * The manager subscribes to `repo.watch({})` and re-emits each event
+   * through {@link notifyWatchers} as a legacy `MetadataWatchEvent`.
+   * Each event also invalidates the in-memory registry entry and the
+   * `list()` cache for the affected type so subsequent reads see fresh
+   * data.
+   *
+   * No write-through. `register()` / `unregister()` / `save()` are
+   * untouched in this PR (deferred to ADR-0008 M0 PR-10).
+   *
+   * Call {@link dispose} (or {@link stopRepositoryWatch}) to detach.
+   */
+  setRepository(repo) {
+    if (this.repository === repo) return;
+    if (this.repository) {
+      void this.stopRepositoryWatch();
+    }
+    this.repository = repo;
+    this.repoWatchClosed = false;
+    void this.startRepositoryWatch();
+  }
+  /** Return the attached repository, if any. */
+  getRepository() {
+    return this.repository;
+  }
+  /** Stop the active repo.watch() loop (best-effort). */
+  async stopRepositoryWatch() {
+    this.repoWatchClosed = true;
+    const iter = this.repoWatchIter;
+    this.repoWatchIter = void 0;
+    if (iter && typeof iter.return === "function") {
+      try {
+        await iter.return(void 0);
+      } catch {
+      }
+    }
+  }
+  /**
+   * Best-effort cleanup. Stops the FS watcher (if any), drains the
+   * repository watch loop, and clears registry caches. Safe to call
+   * multiple times.
+   */
+  async dispose() {
+    await this.stopWatching().catch(() => void 0);
+    await this.stopRepositoryWatch().catch(() => void 0);
+    this.listCache.clear();
+  }
+  async startRepositoryWatch() {
+    const repo = this.repository;
+    if (!repo) return;
+    const iterable = repo.watch({});
+    const iter = iterable[Symbol.asyncIterator]();
+    this.repoWatchIter = iter;
+    try {
+      while (!this.repoWatchClosed) {
+        const { value, done } = await iter.next();
+        if (done) break;
+        try {
+          this.applyRepoEvent(value);
+        } catch (err) {
+          this.logger.warn("[MetadataManager] repo event handler failed", {
+            error: err instanceof Error ? err.message : String(err)
+          });
+        }
+      }
+    } catch (err) {
+      if (!this.repoWatchClosed) {
+        this.logger.warn("[MetadataManager] repository watch loop exited unexpectedly", {
+          error: err instanceof Error ? err.message : String(err)
+        });
+      }
+    } finally {
+      if (this.repoWatchIter === iter) this.repoWatchIter = void 0;
+    }
+  }
+  /** Translate a repo event to the legacy MetadataWatchEvent + invalidate caches. */
+  applyRepoEvent(evt) {
+    const ref = evt.ref;
+    const type = ref.type;
+    const name = ref.name;
+    const typeStore = this.registry.get(type);
+    if (typeStore) {
+      typeStore.delete(name);
+      if (typeStore.size === 0) this.registry.delete(type);
+    }
+    this.listCache.delete(type);
+    const legacyType = evt.op === "create" ? "added" : evt.op === "delete" ? "deleted" : "changed";
+    const legacyEvent = {
+      type: legacyType,
+      metadataType: type,
+      name,
+      path: "",
+      // Repo events carry the hash only; the body is fetched on demand
+      // via manager.get(type, name). HMR consumers don't read `data` so
+      // this is fine for M0. (See ADR-0008 §12 open question 1.)
+      data: void 0,
+      timestamp: evt.ts
+    };
+    legacyEvent.seq = evt.seq;
+    this.notifyWatchers(type, legacyEvent);
+  }
   notifyWatchers(type, event) {
     const callbacks = this.watchCallbacks.get(type);
     if (!callbacks) return;
@@ -2757,7 +2866,13 @@ var NodeMetadataManager = class extends MetadataManager {
     this.watcher = (0, import_chokidar.watch)(rootDir, {
       ignored,
       persistent,
-      ignoreInitial: true
+      ignoreInitial: true,
+      // Use polling to avoid `fs.watch` EMFILE on macOS / busy dev hosts.
+      // Recursive watch over a project root would otherwise wire native
+      // watches across the entire tree, easily exhausting the FD pool.
+      usePolling: true,
+      interval: 1e3,
+      binaryInterval: 2e3
     });
     this.watcher.on("add", async (filePath) => {
       await this.handleFileEvent("added", filePath);
@@ -2892,6 +3007,7 @@ var queryableMetadataObjects = [
   import_metadata2.SysMetadataObject,
   import_metadata2.SysMetadataHistoryObject
 ];
+var REPO_SUBDIR = ".objectstack/metadata";
 var ARTIFACT_FIELD_TO_TYPE = {
   objects: "object",
   objectExtensions: "object_extension",
@@ -2989,6 +3105,31 @@ var MetadataPlugin = class {
           await this._loadFromFileSystem(ctx);
         }
       }
+      const bootstrapMode = this.options.config?.bootstrap ?? "eager";
+      if (bootstrapMode !== "artifact-only") {
+        try {
+          const path3 = await import("path");
+          const { FileSystemRepository } = await import("@objectstack/metadata-fs");
+          const rootDir = this.options.rootDir || process.cwd();
+          const repoRoot = path3.join(rootDir, REPO_SUBDIR);
+          const repo = new FileSystemRepository({
+            root: repoRoot,
+            org: this.options.organizationId ?? "system",
+            disableWatch: this.options.watch === false
+          });
+          await repo.start();
+          this.repository = repo;
+          this.manager.setRepository(repo);
+          ctx.logger.info("[MetadataPlugin] FileSystemRepository attached", {
+            repoRoot,
+            watch: this.options.watch !== false
+          });
+        } catch (e) {
+          ctx.logger.warn("[MetadataPlugin] Failed to attach FileSystemRepository", {
+            error: e?.message
+          });
+        }
+      }
       try {
         const realtimeService = ctx.getService("realtime");
         if (realtimeService && typeof realtimeService === "object" && "publish" in realtimeService) {
@@ -3006,12 +3147,12 @@ var MetadataPlugin = class {
           const { registerMetadataHmrRoutes: registerMetadataHmrRoutes2 } = await Promise.resolve().then(() => (init_hmr_routes(), hmr_routes_exports));
           const hub = registerMetadataHmrRoutes2(httpServer.getRawApp(), this.manager);
           hub.setOnPostReload(async (body = {}) => {
-            const src2 = this.options.artifactSource;
-            if (src2?.mode === "local-file") {
+            const src3 = this.options.artifactSource;
+            if (src3?.mode === "local-file") {
               try {
-                await this._loadFromLocalFile(ctx, src2.path, src2.fetchTimeoutMs);
+                await this._loadFromLocalFile(ctx, src3.path, src3.fetchTimeoutMs);
                 ctx.logger.info("[MetadataPlugin] artifact reloaded via HMR POST", {
-                  path: src2.path,
+                  path: src3.path,
                   reason: body?.reason
                 });
               } catch (e) {
@@ -3020,6 +3161,53 @@ var MetadataPlugin = class {
               }
             }
           });
+          const src2 = this.options.artifactSource;
+          const wantArtifactWatch = this.options.artifactWatch ?? src2?.mode === "local-file";
+          if (src2?.mode === "local-file" && wantArtifactWatch && !/^https?:\/\//i.test(src2.path)) {
+            try {
+              const { watch: chokidarWatch2 } = await import("chokidar");
+              const w = chokidarWatch2(src2.path, {
+                ignoreInitial: true,
+                awaitWriteFinish: { stabilityThreshold: 50, pollInterval: 20 },
+                persistent: true,
+                // Use polling to avoid `fs.watch` exhausting the
+                // process file-descriptor limit on macOS (chokidar
+                // recursively wires watches on the parent
+                // directory tree which can trip EMFILE on busy
+                // dev hosts). 500ms polling is fast enough for
+                // HMR (a recompile takes ~400ms anyway).
+                usePolling: true,
+                interval: 500,
+                binaryInterval: 1e3
+              });
+              let pending = false;
+              const reload = async () => {
+                if (pending) return;
+                pending = true;
+                try {
+                  await this._loadFromLocalFile(ctx, src2.path, src2.fetchTimeoutMs);
+                  hub.broadcastReload("artifact-file-changed", [src2.path]);
+                  ctx.logger.info("[MetadataPlugin] artifact auto-reloaded (file watcher)", {
+                    path: src2.path
+                  });
+                } catch (e) {
+                  ctx.logger.warn("[MetadataPlugin] artifact auto-reload failed", { error: e?.message });
+                } finally {
+                  pending = false;
+                }
+              };
+              w.on("change", () => {
+                void reload();
+              });
+              w.on("add", () => {
+                void reload();
+              });
+              this.artifactWatcher = { close: () => w.close() };
+              console.log("[MetadataPlugin] artifact file watcher attached", src2.path);
+            } catch (e) {
+              ctx.logger.warn("[MetadataPlugin] artifact watcher failed to start", { error: e?.message });
+            }
+          }
           console.log("[MetadataPlugin] HMR endpoint registered at /api/v1/dev/metadata-events");
         } else {
           console.log("[MetadataPlugin] HTTP server with getRawApp() not available \u2014 skipping HMR endpoint");
@@ -3028,6 +3216,28 @@ var MetadataPlugin = class {
         console.warn("[MetadataPlugin] Failed to register HMR endpoint", e?.message);
       }
     };
+    this.stop = async (ctx) => {
+      if (this.artifactWatcher) {
+        try {
+          await this.artifactWatcher.close();
+        } catch {
+        }
+        this.artifactWatcher = void 0;
+      }
+      try {
+        await this.manager.dispose();
+      } catch (e) {
+        ctx.logger.warn("[MetadataPlugin] manager.dispose() failed", { error: e?.message });
+      }
+      const repo = this.repository;
+      if (repo && typeof repo.close === "function") {
+        try {
+          await repo.close();
+        } catch {
+        }
+      }
+      this.repository = void 0;
+    };
     this.options = {
       watch: true,
       ...options
@@ -3104,7 +3314,12 @@ var MetadataPlugin = class {
       const items = metadata[field];
       if (!Array.isArray(items) || items.length === 0) continue;
       for (const item of items) {
-        const name = item?.name;
+        let name = item?.name;
+        if (!name) {
+          if (metaType === "view") {
+            name = item?.list?.data?.object ?? item?.form?.data?.object;
+          }
+        }
         if (!name) continue;
         if (manifestPackageId && item._packageId === void 0) {
           item._packageId = manifestPackageId;
@@ -3413,6 +3628,10 @@ function registerMetadataHistoryRoutes(app, metadataService) {
 }
 
 // src/utils/history-cleanup.ts
+var import_kernel2 = require("@objectstack/spec/kernel");
+function executionPinnedTypes() {
+  return import_kernel2.DEFAULT_METADATA_TYPE_REGISTRY.filter((entry) => entry.executionPinned).map((entry) => entry.type);
+}
 var HistoryCleanupManager = class {
   constructor(policy, dbLoader) {
     this.policy = policy;
@@ -3448,9 +3667,10 @@ var HistoryCleanupManager = class {
     const driver = this.dbLoader.driver;
     const historyTableName = this.dbLoader.historyTableName;
     const organizationId = this.dbLoader.organizationId;
-    const projectId = this.dbLoader.projectId;
     let deleted = 0;
     let errors = 0;
+    const pinnedTypes = executionPinnedTypes();
+    const isPinned = (t) => !!t && pinnedTypes.includes(t);
     try {
       if (this.policy.maxAgeDays) {
         const cutoffDate = /* @__PURE__ */ new Date();
@@ -3462,8 +3682,8 @@ var HistoryCleanupManager = class {
         if (organizationId) {
           filter.organization_id = organizationId;
         }
-        if (projectId !== void 0) {
-          filter.project_id = projectId;
+        if (pinnedTypes.length > 0) {
+          filter.type = { $nin: pinnedTypes };
         }
         try {
           const result = await this.bulkDeleteByFilter(driver, historyTableName, filter);
@@ -3477,20 +3697,22 @@ var HistoryCleanupManager = class {
         try {
           const baseWhere = {};
           if (organizationId) baseWhere.organization_id = organizationId;
-          if (projectId !== void 0) baseWhere.project_id = projectId;
-          const metadataIds = await driver.find(historyTableName, {
+          const metaItems = await driver.find(historyTableName, {
             object: historyTableName,
             where: baseWhere,
-            fields: ["metadata_id"]
+            fields: ["type", "name"]
           });
-          const uniqueIds = /* @__PURE__ */ new Set();
-          for (const record of metadataIds) {
-            if (record.metadata_id) {
-              uniqueIds.add(record.metadata_id);
+          const uniqueKeys = /* @__PURE__ */ new Set();
+          for (const record of metaItems) {
+            const t = record.type;
+            const n = record.name;
+            if (t && n && !isPinned(t)) {
+              uniqueKeys.add(`${t}${n}`);
             }
           }
-          for (const metadataId of uniqueIds) {
-            const filter = { metadata_id: metadataId, ...baseWhere };
+          for (const key of uniqueKeys) {
+            const [type, name] = key.split("");
+            const filter = { type, name, ...baseWhere };
             try {
               const historyRecords = await driver.find(historyTableName, {
                 object: historyTableName,
@@ -3565,13 +3787,13 @@ var HistoryCleanupManager = class {
     const driver = this.dbLoader.driver;
     const historyTableName = this.dbLoader.historyTableName;
     const organizationId = this.dbLoader.organizationId;
-    const projectId = this.dbLoader.projectId;
     let recordsByAge = 0;
     let recordsByCount = 0;
+    const pinnedTypes = executionPinnedTypes();
+    const isPinned = (t) => !!t && pinnedTypes.includes(t);
     try {
       const baseWhere = {};
       if (organizationId) baseWhere.organization_id = organizationId;
-      if (projectId !== void 0) baseWhere.project_id = projectId;
       if (this.policy.maxAgeDays) {
         const cutoffDate = /* @__PURE__ */ new Date();
         cutoffDate.setDate(cutoffDate.getDate() - this.policy.maxAgeDays);
@@ -3580,25 +3802,31 @@ var HistoryCleanupManager = class {
           recorded_at: { $lt: cutoffISO },
           ...baseWhere
         };
+        if (pinnedTypes.length > 0) {
+          filter.type = { $nin: pinnedTypes };
+        }
         recordsByAge = await driver.count(historyTableName, {
           object: historyTableName,
           where: filter
         });
       }
       if (this.policy.maxVersions) {
-        const metadataIds = await driver.find(historyTableName, {
+        const metaItems = await driver.find(historyTableName, {
           object: historyTableName,
           where: baseWhere,
-          fields: ["metadata_id"]
+          fields: ["type", "name"]
         });
-        const uniqueIds = /* @__PURE__ */ new Set();
-        for (const record of metadataIds) {
-          if (record.metadata_id) {
-            uniqueIds.add(record.metadata_id);
+        const uniqueKeys = /* @__PURE__ */ new Set();
+        for (const record of metaItems) {
+          const t = record.type;
+          const n = record.name;
+          if (t && n && !isPinned(t)) {
+            uniqueKeys.add(`${t}${n}`);
           }
         }
-        for (const metadataId of uniqueIds) {
-          const filter = { metadata_id: metadataId, ...baseWhere };
+        for (const key of uniqueKeys) {
+          const [type, name] = key.split("");
+          const filter = { type, name, ...baseWhere };
           const count = await driver.count(historyTableName, {
             object: historyTableName,
             where: filter