@mastra/mongodb 1.11.1 → 1.12.0-alpha.0

package/dist/index.js

@@ -1,6 +1,6 @@
 import { v4 } from '@lukeed/uuid';
 import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
-import { createVectorErrorId, AgentsStorage, TABLE_AGENTS, TABLE_AGENT_VERSIONS, createStorageErrorId, normalizePerPage, calculatePagination, BackgroundTasksStorage, TABLE_BACKGROUND_TASKS, BlobStore, TABLE_SKILL_BLOBS, DatasetsStorage, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, ensureDate, safelyParseJSON, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, ExperimentsStorage, MCPClientsStorage, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, MCPServersStorage, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, MemoryStorage, TABLE_THREADS, TABLE_MESSAGES, TABLE_RESOURCES, NotificationsStorage, TABLE_NOTIFICATIONS, ObservabilityStorage, TABLE_SPANS, listTracesArgsSchema, toTraceSpans, PromptBlocksStorage, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, SchedulesStorage, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, ScorerDefinitionsStorage, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, ScoresStorage, TABLE_SCORERS, SkillsStorage, TABLE_SKILLS, TABLE_SKILL_VERSIONS, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, WorkspacesStorage, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, MastraCompositeStore, TraceStatus, transformScoreRow as transformScoreRow$1 } from '@mastra/core/storage';
+import { createVectorErrorId, AgentsStorage, TABLE_AGENTS, TABLE_AGENT_VERSIONS, createStorageErrorId, normalizePerPage, calculatePagination, BackgroundTasksStorage, TABLE_BACKGROUND_TASKS, BlobStore, TABLE_SKILL_BLOBS, DatasetsStorage, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, ensureDate, safelyParseJSON, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, ExperimentsStorage, MCPClientsStorage, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, MCPServersStorage, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, MemoryStorage, TABLE_THREADS, TABLE_MESSAGES, TABLE_RESOURCES, NotificationsStorage, TABLE_NOTIFICATIONS, ObservabilityStorage, TABLE_SPANS, listTracesArgsSchema, toTraceSpans, PromptBlocksStorage, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, SchedulesStorage, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, ScorerDefinitionsStorage, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, ScoresStorage, TABLE_SCORERS, SkillsStorage, TABLE_SKILLS, TABLE_SKILL_VERSIONS, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, WorkspacesStorage, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, MastraCompositeStore, TraceStatus, parseDuration, transformScoreRow as transformScoreRow$1 } from '@mastra/core/storage';
 import { MastraVector, validateUpsertInput, validateVectorValues } from '@mastra/core/vector';
 import { MongoClient, MongoBulkWriteError } from 'mongodb';
 import { BaseFilterTranslator } from '@mastra/core/vector/filter';
@@ -13,7 +13,7 @@ import { skillSnapshotFieldValuesEqual } from '@mastra/core/storage/domains/skil
 
 // package.json
 var package_default = {
-  version: "1.11.1"};
+  version: "1.12.0-alpha.0"};
 var MongoDBFilterTranslator = class extends BaseFilterTranslator {
   getSupportedOperators() {
     return {
@@ -1619,6 +1619,126 @@ var MongoDBAgentsStorage = class _MongoDBAgentsStorage extends AgentsStorage {
     return result;
   }
 };
+var DEFAULT_PRUNE_BATCH_SIZE = 1e3;
+async function ensureAnchorIndex(connector, target, logger) {
+  if (!target.indexed) return;
+  try {
+    const collection = await connector.getCollection(target.table);
+    await collection.createIndex({ [target.column]: -1 });
+  } catch (error) {
+    logger?.warn?.(`Failed to ensure retention index on ${target.table}(${target.column}):`, error);
+  }
+}
+async function sleep(ms, signal) {
+  if (ms <= 0 || signal?.aborted) return;
+  await new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      resolve();
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+function cutoffFor(policy, encoding, now = Date.now()) {
+  const cutoffMs = now - parseDuration(policy.maxAge);
+  if (encoding === "epoch-ms") return cutoffMs;
+  if (encoding === "iso") return new Date(cutoffMs).toISOString();
+  return new Date(cutoffMs);
+}
+async function pruneCollectionBatch({
+  connector,
+  table,
+  column,
+  cutoff,
+  limit
+}) {
+  const collection = await connector.getCollection(table);
+  const docs = await collection.find({ [column]: { $lt: cutoff } }).project({ _id: 1 }).limit(limit).toArray();
+  if (docs.length === 0) return 0;
+  const result = await collection.deleteMany({ _id: { $in: docs.map((doc) => doc._id) } });
+  return result.deletedCount;
+}
+async function runBatchedDelete({
+  deleteBatch,
+  batchSize,
+  options
+}) {
+  if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {
+    throw new Error(`retention batchSize must be a positive integer; received ${batchSize}`);
+  }
+  let deleted = 0;
+  let batches = 0;
+  while (true) {
+    if (options?.signal?.aborted) return { deleted, done: false };
+    if (options?.maxBatches !== void 0 && batches >= options.maxBatches) return { deleted, done: false };
+    let limit = batchSize;
+    if (options?.maxRows !== void 0) {
+      const remaining = options.maxRows - deleted;
+      if (remaining <= 0) return { deleted, done: false };
+      limit = Math.min(limit, remaining);
+    }
+    const affected = await deleteBatch(limit);
+    deleted += affected;
+    batches += 1;
+    if (affected < limit) return { deleted, done: true };
+    if (options?.pauseMs) {
+      await sleep(options.pauseMs, options.signal);
+    }
+  }
+}
+async function runPrune({
+  connector,
+  domain,
+  targets,
+  options,
+  logger
+}) {
+  const results = [];
+  const now = Date.now();
+  for (const target of targets) {
+    if (options?.signal?.aborted) {
+      results.push({ domain, table: target.table, deleted: 0, done: false });
+      continue;
+    }
+    await ensureAnchorIndex(connector, target, logger);
+    const cutoff = cutoffFor(target.policy, target.encoding, now);
+    const batchSize = target.policy.batchSize ?? DEFAULT_PRUNE_BATCH_SIZE;
+    const { deleted, done } = await runBatchedDelete({
+      deleteBatch: (limit) => pruneCollectionBatch({ connector, table: target.table, column: target.column, cutoff, limit }),
+      batchSize,
+      options
+    });
+    results.push({ domain, table: target.table, deleted, done });
+  }
+  return results;
+}
+function resolveTargets({
+  policies,
+  descriptor,
+  order,
+  encodings
+}) {
+  const targets = [];
+  for (const key of order) {
+    const policy = policies[key];
+    const entry = descriptor[key];
+    if (!policy || !entry) continue;
+    targets.push({
+      table: entry.table,
+      column: entry.column,
+      encoding: encodings?.[key] ?? (entry.anchorType === "epoch-ms" ? "epoch-ms" : "date"),
+      indexed: entry.indexed ?? true,
+      policy
+    });
+  }
+  return targets;
+}
+
+// src/storage/domains/background-tasks/index.ts
 function toDoc(task) {
   return {
     id: task.id,
@@ -1670,6 +1790,14 @@ var BackgroundTasksStorageMongoDB = class _BackgroundTasksStorageMongoDB extends
   #skipDefaultIndexes;
   #indexes;
   static MANAGED_COLLECTIONS = [TABLE_BACKGROUND_TASKS];
+  /**
+   * Anchor is `completedAt`, so age is measured from task completion; it stays
+   * `null` while a task is in-flight, and `null $lt cutoff` never matches, so
+   * pending/running tasks are never pruned regardless of age.
+   */
+  static retentionTables = {
+    backgroundTasks: { table: TABLE_BACKGROUND_TASKS, column: "completedAt", indexed: true }
+  };
   constructor(config) {
     super();
     this.#connector = resolveMongoDBConfig(config);
@@ -1681,6 +1809,20 @@ var BackgroundTasksStorageMongoDB = class _BackgroundTasksStorageMongoDB extends
   async getCollection() {
     return this.#connector.getCollection(TABLE_BACKGROUND_TASKS);
   }
+  /**
+   * Delete tasks completed longer ago than the policy's `maxAge`, batched.
+   * `completedAt` is stored as an ISO-8601 string (see `toDoc`), so the cutoff
+   * uses the `iso` encoding — lexicographic `$lt` on ISO strings is time-ordered.
+   */
+  async prune(policies, options) {
+    const targets = resolveTargets({
+      policies,
+      descriptor: _BackgroundTasksStorageMongoDB.retentionTables,
+      order: ["backgroundTasks"],
+      encodings: { backgroundTasks: "iso" }
+    });
+    return runPrune({ connector: this.#connector, domain: "backgroundTasks", targets, options, logger: this.logger });
+  }
   getDefaultIndexDefinitions() {
     return [
       { collection: TABLE_BACKGROUND_TASKS, keys: { id: 1 }, options: { unique: true } },
@@ -3081,6 +3223,15 @@ var MongoDBExperimentsStorage = class _MongoDBExperimentsStorage extends Experim
   #skipDefaultIndexes;
   #indexes;
   static MANAGED_COLLECTIONS = [TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS];
+  /**
+   * Experiments prune as whole units: an experiment and its results are only
+   * deleted together, once the experiment itself is old (anchored on the
+   * parent's `completedAt`, a BSON date that stays `null` while running).
+   * `results` is intentionally not an independent retention key.
+   */
+  static retentionTables = {
+    experiments: { table: TABLE_EXPERIMENTS, column: "completedAt", indexed: true }
+  };
   constructor(config) {
     super();
     this.#connector = resolveMongoDBConfig(config);
@@ -3092,6 +3243,57 @@ var MongoDBExperimentsStorage = class _MongoDBExperimentsStorage extends Experim
   async getCollection(name) {
     return this.#connector.getCollection(name);
   }
+  /**
+   * Prune whole experiments older than the `experiments` policy's `maxAge`.
+   *
+   * Each batch collects up to `batchSize` aged experiment ids
+   * (`completedAt < cutoff`; BSON type bracketing means `null` — still
+   * running — never matches), then deletes their `experiment_results` rows and
+   * the experiment rows for exactly that id set inside
+   * `connector.withTransaction()` — atomic on replica sets, sequential
+   * children-first on standalone — mirroring `deleteExperiment`. Hitting
+   * `maxBatches`/`maxRows` or the abort signal between batches therefore never
+   * leaves a run hollow (parent kept, results gone). Bounds count whole
+   * experiments, not rows.
+   */
+  async prune(policies, options) {
+    const policy = policies["experiments"];
+    if (!policy || options?.signal?.aborted) {
+      return policy ? [
+        { domain: "experiments", table: TABLE_EXPERIMENT_RESULTS, deleted: 0, done: false },
+        { domain: "experiments", table: TABLE_EXPERIMENTS, deleted: 0, done: false }
+      ] : [];
+    }
+    await ensureAnchorIndex(
+      this.#connector,
+      { table: TABLE_EXPERIMENTS, column: "completedAt", indexed: true },
+      this.logger
+    );
+    const cutoff = cutoffFor(policy, "date");
+    const batchSize = policy.batchSize ?? DEFAULT_PRUNE_BATCH_SIZE;
+    const experimentsCollection = await this.getCollection(TABLE_EXPERIMENTS);
+    const resultsCollection = await this.getCollection(TABLE_EXPERIMENT_RESULTS);
+    let childDeleted = 0;
+    const parent = await runBatchedDelete({
+      deleteBatch: async (limit) => {
+        const docs = await experimentsCollection.find({ completedAt: { $lt: cutoff } }).project({ id: 1 }).limit(limit).toArray();
+        if (docs.length === 0) return 0;
+        const ids = docs.map((doc) => doc.id);
+        return this.#connector.withTransaction(async (session) => {
+          const children = await resultsCollection.deleteMany({ experimentId: { $in: ids } }, { session });
+          childDeleted += children.deletedCount;
+          const parents = await experimentsCollection.deleteMany({ id: { $in: ids } }, { session });
+          return parents.deletedCount;
+        });
+      },
+      batchSize,
+      options
+    });
+    return [
+      { domain: "experiments", table: TABLE_EXPERIMENT_RESULTS, deleted: childDeleted, done: parent.done },
+      { domain: "experiments", table: TABLE_EXPERIMENTS, deleted: parent.deleted, done: parent.done }
+    ];
+  }
   // -------------------------------------------------------------------------
   // Index Management
   // -------------------------------------------------------------------------
@@ -4689,6 +4891,15 @@ var MemoryStorageMongoDB = class _MemoryStorageMongoDB extends MemoryStorage {
   #indexes;
   /** Collections managed by this domain */
   static MANAGED_COLLECTIONS = [TABLE_THREADS, TABLE_MESSAGES, TABLE_RESOURCES, OM_TABLE];
+  /**
+   * Retention-eligible collections. The observational-memory collection is
+   * excluded: it has no timestamp anchor to age on. All anchors are BSON dates.
+   */
+  static retentionTables = {
+    messages: { table: TABLE_MESSAGES, column: "createdAt", indexed: true },
+    resources: { table: TABLE_RESOURCES, column: "createdAt", indexed: true },
+    threads: { table: TABLE_THREADS, column: "createdAt", indexed: true }
+  };
   constructor(config) {
     super();
     this.#connector = resolveMongoDBConfig(config);
@@ -4704,6 +4915,21 @@ var MemoryStorageMongoDB = class _MemoryStorageMongoDB extends MemoryStorage {
     await this.createDefaultIndexes();
     await this.createCustomIndexes();
   }
+  /**
+   * Delete memory rows older than each table's `maxAge`, batched. Order is
+   * messages → resources → threads so child rows never outlive the delete of
+   * their thread. Like `deleteThread()`, this does not sweep vector-store
+   * embeddings — semantic-recall vectors live in a separate vector store the
+   * memory domain cannot reach; cleaning those up is the operator's concern.
+   */
+  async prune(policies, options) {
+    const targets = resolveTargets({
+      policies,
+      descriptor: _MemoryStorageMongoDB.retentionTables,
+      order: ["messages", "resources", "threads"]
+    });
+    return runPrune({ connector: this.#connector, domain: "memory", targets, options, logger: this.logger });
+  }
   /**
    * Returns default index definitions for the memory domain collections.
    */
@@ -6657,6 +6883,10 @@ var NotificationsMongoDB = class _NotificationsMongoDB extends NotificationsStor
   #skipDefaultIndexes;
   #indexes;
   static MANAGED_COLLECTIONS = [TABLE_NOTIFICATIONS];
+  /** Anchor is the notification's creation time, stored as a BSON date. */
+  static retentionTables = {
+    notifications: { table: TABLE_NOTIFICATIONS, column: "createdAt", indexed: true }
+  };
   constructor(config) {
     super();
     this.#connector = resolveMongoDBConfig(config);
@@ -6668,6 +6898,15 @@ var NotificationsMongoDB = class _NotificationsMongoDB extends NotificationsStor
   async getCollection() {
     return this.#connector.getCollection(TABLE_NOTIFICATIONS);
   }
+  /** Delete notifications older than the policy's `maxAge`, batched. */
+  async prune(policies, options) {
+    const targets = resolveTargets({
+      policies,
+      descriptor: _NotificationsMongoDB.retentionTables,
+      order: ["notifications"]
+    });
+    return runPrune({ connector: this.#connector, domain: "notifications", targets, options, logger: this.logger });
+  }
   async init() {
     await this.createDefaultIndexes();
     await this.createCustomIndexes();
@@ -6860,6 +7099,10 @@ var ObservabilityMongoDB = class _ObservabilityMongoDB extends ObservabilityStor
   #indexes;
   /** Collections managed by this domain */
   static MANAGED_COLLECTIONS = [TABLE_SPANS];
+  /** Anchor is the span's start time, stored as a BSON date. */
+  static retentionTables = {
+    spans: { table: TABLE_SPANS, column: "startedAt", indexed: true }
+  };
   constructor(config) {
     super();
     this.#connector = resolveMongoDBConfig(config);
@@ -6871,6 +7114,15 @@ var ObservabilityMongoDB = class _ObservabilityMongoDB extends ObservabilityStor
   async getCollection(name) {
     return this.#connector.getCollection(name);
   }
+  /** Delete spans older than the `spans` policy's `maxAge`, batched. */
+  async prune(policies, options) {
+    const targets = resolveTargets({
+      policies,
+      descriptor: _ObservabilityMongoDB.retentionTables,
+      order: ["spans"]
+    });
+    return runPrune({ connector: this.#connector, domain: "observability", targets, options, logger: this.logger });
+  }
   /**
    * Returns default index definitions for the observability domain collections.
    * These indexes optimize common query patterns for span and trace lookups.
@@ -8285,6 +8537,16 @@ var SchedulesMongoDB = class _SchedulesMongoDB extends SchedulesStorage {
   #skipDefaultIndexes;
   #indexes;
   static MANAGED_COLLECTIONS = [TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS];
+  /**
+   * Only trigger (fire-history) rows are retention-eligible — schedule
+   * definitions are config, not growth. Anchor `actual_fire_at` is stored as a
+   * raw epoch-ms number, so `anchorType: 'epoch-ms'` keeps the cutoff numeric.
+   * The default composite index `(schedule_id, actual_fire_at)` can't serve a
+   * bare `actual_fire_at` range scan, hence the lazy single-field anchor index.
+   */
+  static retentionTables = {
+    triggers: { table: TABLE_SCHEDULE_TRIGGERS, column: "actual_fire_at", anchorType: "epoch-ms", indexed: true }
+  };
   constructor(config) {
     super();
     this.#connector = resolveMongoDBConfig(config);
@@ -8299,6 +8561,15 @@ var SchedulesMongoDB = class _SchedulesMongoDB extends SchedulesStorage {
   getTriggersCollection() {
     return this.#connector.getCollection(TABLE_SCHEDULE_TRIGGERS);
   }
+  /** Delete trigger (fire-history) rows older than the `triggers` policy's `maxAge`, batched. */
+  async prune(policies, options) {
+    const targets = resolveTargets({
+      policies,
+      descriptor: _SchedulesMongoDB.retentionTables,
+      order: ["triggers"]
+    });
+    return runPrune({ connector: this.#connector, domain: "schedules", targets, options, logger: this.logger });
+  }
   getDefaultIndexDefinitions() {
     return [
       { collection: TABLE_SCHEDULES, keys: { id: 1 }, options: { unique: true } },
@@ -9002,6 +9273,10 @@ var ScoresStorageMongoDB = class _ScoresStorageMongoDB extends ScoresStorage {
   #indexes;
   /** Collections managed by this domain */
   static MANAGED_COLLECTIONS = [TABLE_SCORERS];
+  /** Anchor is the score row's creation time, stored as a BSON date. */
+  static retentionTables = {
+    scorers: { table: TABLE_SCORERS, column: "createdAt", indexed: true }
+  };
   constructor(config) {
     super();
     this.#connector = resolveMongoDBConfig(config);
@@ -9013,6 +9288,15 @@ var ScoresStorageMongoDB = class _ScoresStorageMongoDB extends ScoresStorage {
   async getCollection(name) {
     return this.#connector.getCollection(name);
   }
+  /** Delete scorer results older than the `scorers` policy's `maxAge`, batched. */
+  async prune(policies, options) {
+    const targets = resolveTargets({
+      policies,
+      descriptor: _ScoresStorageMongoDB.retentionTables,
+      order: ["scorers"]
+    });
+    return runPrune({ connector: this.#connector, domain: "scores", targets, options, logger: this.logger });
+  }
   /**
    * Returns default index definitions for the scores domain collections.
    * These indexes optimize common query patterns for score lookups.
@@ -9980,6 +10264,13 @@ var WorkflowsStorageMongoDB = class _WorkflowsStorageMongoDB extends WorkflowsSt
   #indexes;
   /** Collections managed by this domain */
   static MANAGED_COLLECTIONS = [TABLE_WORKFLOW_SNAPSHOT];
+  /**
+   * Anchor is `updatedAt` (BSON date), so the policy reads as inactivity:
+   * a run is pruned only after its snapshot has not been touched for `maxAge`.
+   */
+  static retentionTables = {
+    workflowSnapshot: { table: TABLE_WORKFLOW_SNAPSHOT, column: "updatedAt", indexed: true }
+  };
   constructor(config) {
     super();
     this.#connector = resolveMongoDBConfig(config);
@@ -9988,6 +10279,15 @@ var WorkflowsStorageMongoDB = class _WorkflowsStorageMongoDB extends WorkflowsSt
       (idx) => _WorkflowsStorageMongoDB.MANAGED_COLLECTIONS.includes(idx.collection)
     );
   }
+  /** Delete workflow snapshots idle for longer than the policy's `maxAge`, batched. */
+  async prune(policies, options) {
+    const targets = resolveTargets({
+      policies,
+      descriptor: _WorkflowsStorageMongoDB.retentionTables,
+      order: ["workflowSnapshot"]
+    });
+    return runPrune({ connector: this.#connector, domain: "workflows", targets, options, logger: this.logger });
+  }
   supportsConcurrentUpdates() {
     return true;
   }
@@ -10995,7 +11295,7 @@ var MongoDBStore = class extends MastraCompositeStore {
   #connector;
   stores;
   constructor(config) {
-    super({ id: config.id, name: "MongoDBStore", disableInit: config.disableInit });
+    super({ id: config.id, name: "MongoDBStore", disableInit: config.disableInit, retention: config.retention });
     this.#connector = resolveMongoDBConfig(config);
     const domainConfig = {
       connector: this.#connector,