@mastra/mongodb 1.23.0-alpha.1 → 1.23.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_types/@internal_core/dist/memory/index.d.ts +102 -0
- package/dist/_types/@internal_core/package.json +1 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/reference-storage-retention.md +21 -18
- package/dist/index.cjs +618 -292
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +618 -292
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/memory/index.d.ts +3 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/package.json +7 -6
package/dist/index.js
CHANGED
|
@@ -4,11 +4,12 @@ import { AgentsStorage, BackgroundTasksStorage, BlobStore, DatasetsStorage, Expe
|
|
|
4
4
|
import { MastraVector, validateUpsertInput, validateVectorValues } from "@mastra/core/vector";
|
|
5
5
|
import { MongoBulkWriteError, MongoClient, ObjectId } from "mongodb";
|
|
6
6
|
import { BaseFilterTranslator } from "@mastra/core/vector/filter";
|
|
7
|
+
import { createHash } from "crypto";
|
|
7
8
|
import { MessageList } from "@mastra/core/agent";
|
|
8
9
|
import { saveScorePayloadSchema } from "@mastra/core/evals";
|
|
9
10
|
import { skillSnapshotFieldValuesEqual } from "@mastra/core/storage/domains/skills";
|
|
10
11
|
//#region package.json
|
|
11
|
-
var version = "1.23.
|
|
12
|
+
var version = "1.23.1-alpha.0";
|
|
12
13
|
//#endregion
|
|
13
14
|
//#region src/vector/filter.ts
|
|
14
15
|
/**
|
|
@@ -6494,6 +6495,71 @@ var MongoDBMCPServersStorage = class MongoDBMCPServersStorage extends MCPServers
|
|
|
6494
6495
|
}
|
|
6495
6496
|
};
|
|
6496
6497
|
//#endregion
|
|
6498
|
+
//#region ../../packages/_internal-core/dist/memory/index.js
|
|
6499
|
+
/**
|
|
6500
|
+
* A buffered chunk stores `lastObservedAt = max message time + 1ms`. It is wholly covered
|
|
6501
|
+
* (every message it observed is already at or before the cursor) iff
|
|
6502
|
+
* `cursor >= chunk.lastObservedAt - 1ms`.
|
|
6503
|
+
*/
|
|
6504
|
+
function isBufferedChunkCoveredByCursor(chunkLastObservedAt, cursor) {
|
|
6505
|
+
if (!cursor) return false;
|
|
6506
|
+
return new Date(cursor).getTime() >= new Date(chunkLastObservedAt).getTime() - 1;
|
|
6507
|
+
}
|
|
6508
|
+
/** The later of two cursors; `undefined` only when both are absent. */
|
|
6509
|
+
function maxObservationCursor(a, b) {
|
|
6510
|
+
if (!a) return b ? new Date(b) : void 0;
|
|
6511
|
+
if (!b) return new Date(a);
|
|
6512
|
+
return new Date(Math.max(new Date(a).getTime(), new Date(b).getTime()));
|
|
6513
|
+
}
|
|
6514
|
+
/**
|
|
6515
|
+
* Decide the new generation's text when a reflection built from `snapshot` commits against
|
|
6516
|
+
* the stored record.
|
|
6517
|
+
*
|
|
6518
|
+
* - Stored text equals the snapshot: the reflection replaces it.
|
|
6519
|
+
* - Stored text extends the snapshot (activation only appends): the appended tail is kept
|
|
6520
|
+
* after the reflection, and its tokens are added.
|
|
6521
|
+
* - Anything else (a non-append rewrite): `null` — the reflection must not be applied.
|
|
6522
|
+
*/
|
|
6523
|
+
function planReflectionGenerationText(input) {
|
|
6524
|
+
const stored = input.storedObservations ?? "";
|
|
6525
|
+
const snapshot = input.snapshotObservations ?? "";
|
|
6526
|
+
if (stored === snapshot) return {
|
|
6527
|
+
mode: "equal",
|
|
6528
|
+
observations: input.reflection,
|
|
6529
|
+
tokenCount: input.tokenCount
|
|
6530
|
+
};
|
|
6531
|
+
if (!stored.startsWith(snapshot)) return null;
|
|
6532
|
+
const tail = stored.slice(snapshot.length).trimStart();
|
|
6533
|
+
if (!tail.trim()) return {
|
|
6534
|
+
mode: "equal",
|
|
6535
|
+
observations: input.reflection,
|
|
6536
|
+
tokenCount: input.tokenCount
|
|
6537
|
+
};
|
|
6538
|
+
return {
|
|
6539
|
+
mode: "append",
|
|
6540
|
+
observations: input.reflection ? `${input.reflection}\n\n${tail}` : tail,
|
|
6541
|
+
tokenCount: input.tokenCount + Math.max(0, (input.storedObservationTokenCount ?? 0) - (input.snapshotObservationTokenCount ?? 0))
|
|
6542
|
+
};
|
|
6543
|
+
}
|
|
6544
|
+
/**
|
|
6545
|
+
* Backs `updateActiveObservations` in adapters that implement `commitActiveObservations`.
|
|
6546
|
+
* Callers of `updateActiveObservations` (which resolves `void`) can't see a commit result, so a
|
|
6547
|
+
* write that was not applied throws instead of resolving as if it had committed.
|
|
6548
|
+
*/
|
|
6549
|
+
function assertActiveObservationsApplied(result, id) {
|
|
6550
|
+
if (result.applied) return;
|
|
6551
|
+
throw new Error(result.reason === "retired" ? `Observational memory record ${id} was superseded by a newer generation; observations were not written` : `Observational memory record ${id} changed since the observations were composed; observations were not written`);
|
|
6552
|
+
}
|
|
6553
|
+
/**
|
|
6554
|
+
* Whether the stored text can still accept a reflection built from `snapshot`
|
|
6555
|
+
* (equal, or only appended to since).
|
|
6556
|
+
*/
|
|
6557
|
+
function isAppendOnlySince(storedObservations, snapshotObservations) {
|
|
6558
|
+
const stored = storedObservations ?? "";
|
|
6559
|
+
const snapshot = snapshotObservations ?? "";
|
|
6560
|
+
return stored === snapshot || stored.startsWith(snapshot);
|
|
6561
|
+
}
|
|
6562
|
+
//#endregion
|
|
6497
6563
|
//#region src/storage/domains/memory/index.ts
|
|
6498
6564
|
/**
|
|
6499
6565
|
* Local constant for the observational memory table name.
|
|
@@ -6501,6 +6567,98 @@ var MongoDBMCPServersStorage = class MongoDBMCPServersStorage extends MCPServers
|
|
|
6501
6567
|
* versions that don't export TABLE_OBSERVATIONAL_MEMORY.
|
|
6502
6568
|
*/
|
|
6503
6569
|
const OM_TABLE = "mastra_observational_memory";
|
|
6570
|
+
/** Canonical head order: newest generation, then earliest createdAt, then lowest id. */
|
|
6571
|
+
const OM_HEAD_SORT = {
|
|
6572
|
+
generationCount: -1,
|
|
6573
|
+
createdAt: 1,
|
|
6574
|
+
id: 1
|
|
6575
|
+
};
|
|
6576
|
+
/** Max redirects when a write aimed at a retired record follows the head. */
|
|
6577
|
+
const OM_MAX_HEAD_HOPS = 3;
|
|
6578
|
+
/** Max read-decide-conditional-update rounds before giving up on a contended record. */
|
|
6579
|
+
const OM_MAX_CONDITIONAL_ATTEMPTS = 3;
|
|
6580
|
+
/** Document `_id` of a key's generation-0 record; see `initializeObservationalMemory`. */
|
|
6581
|
+
function getGeneration0DocumentId(lookupKey) {
|
|
6582
|
+
return `om0_${createHash("sha256").update(lookupKey).digest("hex").slice(0, 32)}`;
|
|
6583
|
+
}
|
|
6584
|
+
/**
|
|
6585
|
+
* The successor document of a fenced (frozen) document: a pure function of the old document's
|
|
6586
|
+
* final state and the rollover inputs, so concurrent roll-forwards insert identical documents.
|
|
6587
|
+
* Buffered chunks, the cursor, buffering markers, flags, and counters carry over; buffered
|
|
6588
|
+
* reflection state does not.
|
|
6589
|
+
*/
|
|
6590
|
+
function buildSuccessorDocument(old, pending) {
|
|
6591
|
+
let activeObservations = pending.reflection;
|
|
6592
|
+
let observationTokenCount = pending.tokenCount;
|
|
6593
|
+
if (pending.mode === "append") {
|
|
6594
|
+
const tail = String(old.activeObservations ?? "").slice(pending.snapTextLength).trimStart();
|
|
6595
|
+
activeObservations = pending.reflection ? `${pending.reflection}\n\n${tail}` : tail;
|
|
6596
|
+
observationTokenCount = pending.tokenCount + Math.max(0, Number(old.observationTokenCount || 0) - pending.snapObservationTokenCount);
|
|
6597
|
+
}
|
|
6598
|
+
return {
|
|
6599
|
+
id: pending.newId,
|
|
6600
|
+
lookupKey: old.lookupKey,
|
|
6601
|
+
scope: old.scope,
|
|
6602
|
+
resourceId: old.resourceId,
|
|
6603
|
+
threadId: old.threadId ?? null,
|
|
6604
|
+
activeObservations,
|
|
6605
|
+
activeObservationsPendingUpdate: null,
|
|
6606
|
+
originType: "reflection",
|
|
6607
|
+
config: old.config ?? null,
|
|
6608
|
+
generationCount: Number(old.generationCount || 0) + 1,
|
|
6609
|
+
lastObservedAt: old.lastObservedAt ?? null,
|
|
6610
|
+
lastReflectionAt: pending.createdAt,
|
|
6611
|
+
pendingMessageTokens: old.pendingMessageTokens ?? 0,
|
|
6612
|
+
totalTokensObserved: old.totalTokensObserved ?? 0,
|
|
6613
|
+
observationTokenCount,
|
|
6614
|
+
bufferedObservationChunks: Array.isArray(old.bufferedObservationChunks) ? old.bufferedObservationChunks : [],
|
|
6615
|
+
isObserving: false,
|
|
6616
|
+
isReflecting: false,
|
|
6617
|
+
isBufferingObservation: Boolean(old.isBufferingObservation),
|
|
6618
|
+
isBufferingReflection: false,
|
|
6619
|
+
lastBufferedAtTokens: old.lastBufferedAtTokens ?? 0,
|
|
6620
|
+
lastBufferedAtTime: old.lastBufferedAtTime ?? null,
|
|
6621
|
+
observedTimezone: old.observedTimezone ?? null,
|
|
6622
|
+
metadata: old.metadata ?? null,
|
|
6623
|
+
supersededBy: null,
|
|
6624
|
+
createdAt: pending.createdAt,
|
|
6625
|
+
updatedAt: pending.createdAt
|
|
6626
|
+
};
|
|
6627
|
+
}
|
|
6628
|
+
/** Equality filters for `fields` as stored in `doc` (`$exists: false` for absent fields). */
|
|
6629
|
+
function matchStoredFields(doc, fields) {
|
|
6630
|
+
return Object.fromEntries(fields.map((f) => [f, doc[f] === void 0 ? { $exists: false } : doc[f]]));
|
|
6631
|
+
}
|
|
6632
|
+
function isDuplicateKeyError(error) {
|
|
6633
|
+
return error?.code === 11e3;
|
|
6634
|
+
}
|
|
6635
|
+
function omNotFound(operation, id) {
|
|
6636
|
+
return new MastraError({
|
|
6637
|
+
id: createStorageErrorId("MONGODB", operation, "NOT_FOUND"),
|
|
6638
|
+
text: `Observational memory record not found: ${id}`,
|
|
6639
|
+
domain: ErrorDomain.STORAGE,
|
|
6640
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
6641
|
+
details: { id }
|
|
6642
|
+
});
|
|
6643
|
+
}
|
|
6644
|
+
function omConflict(operation, id) {
|
|
6645
|
+
return new MastraError({
|
|
6646
|
+
id: createStorageErrorId("MONGODB", operation, "CONFLICT"),
|
|
6647
|
+
text: `Observational memory record ${id} kept changing during ${operation}; giving up after ${OM_MAX_CONDITIONAL_ATTEMPTS} attempts`,
|
|
6648
|
+
domain: ErrorDomain.STORAGE,
|
|
6649
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
6650
|
+
details: { id }
|
|
6651
|
+
});
|
|
6652
|
+
}
|
|
6653
|
+
function omNoLiveHead(id) {
|
|
6654
|
+
return new MastraError({
|
|
6655
|
+
id: createStorageErrorId("MONGODB", "RESOLVE_OBSERVATIONAL_MEMORY_HEAD", "FAILED"),
|
|
6656
|
+
text: `Observational memory record ${id} is superseded but no live head was found`,
|
|
6657
|
+
domain: ErrorDomain.STORAGE,
|
|
6658
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
6659
|
+
details: { id }
|
|
6660
|
+
});
|
|
6661
|
+
}
|
|
6504
6662
|
var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
6505
6663
|
supportsPartialThreadUpdate = true;
|
|
6506
6664
|
supportsObservationalMemory = true;
|
|
@@ -6548,6 +6706,8 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
6548
6706
|
async init() {
|
|
6549
6707
|
await this.createDefaultIndexes();
|
|
6550
6708
|
await this.createCustomIndexes();
|
|
6709
|
+
await this.#warnIfOMIdIndexMissing();
|
|
6710
|
+
await this.#maintainSupersededBy();
|
|
6551
6711
|
}
|
|
6552
6712
|
/**
|
|
6553
6713
|
* Delete memory rows older than each table's `maxAge`, batched. Order is
|
|
@@ -7525,15 +7685,15 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
7525
7685
|
config: doc.config || {},
|
|
7526
7686
|
metadata: doc.metadata || void 0,
|
|
7527
7687
|
observedMessageIds: doc.observedMessageIds || void 0,
|
|
7528
|
-
observedTimezone: doc.observedTimezone || void 0
|
|
7688
|
+
observedTimezone: doc.observedTimezone || void 0,
|
|
7689
|
+
supersededBy: doc.supersededBy ?? null
|
|
7529
7690
|
};
|
|
7530
7691
|
}
|
|
7531
7692
|
async getObservationalMemory(threadId, resourceId) {
|
|
7532
7693
|
try {
|
|
7533
|
-
const
|
|
7534
|
-
const doc = await (
|
|
7535
|
-
|
|
7536
|
-
return this.parseOMDocument(doc);
|
|
7694
|
+
const collection = await this.getCollection(OM_TABLE);
|
|
7695
|
+
const doc = await this.#getHeadDoc(collection, this.getOMKey(threadId, resourceId));
|
|
7696
|
+
return doc ? this.parseOMDocument(doc) : null;
|
|
7537
7697
|
} catch (error) {
|
|
7538
7698
|
throw new MastraError({
|
|
7539
7699
|
id: createStorageErrorId("MONGODB", "GET_OBSERVATIONAL_MEMORY", "FAILED"),
|
|
@@ -7596,60 +7756,55 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
7596
7756
|
}
|
|
7597
7757
|
async initializeObservationalMemory(input) {
|
|
7598
7758
|
try {
|
|
7759
|
+
const lookupKey = this.getOMKey(input.threadId, input.resourceId);
|
|
7760
|
+
const collection = await this.getCollection(OM_TABLE);
|
|
7761
|
+
const existing = await this.#getHeadDoc(collection, lookupKey);
|
|
7762
|
+
if (existing) return this.parseOMDocument(existing);
|
|
7599
7763
|
const id = globalThis.crypto.randomUUID();
|
|
7600
7764
|
const now = /* @__PURE__ */ new Date();
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
|
|
7606
|
-
|
|
7607
|
-
|
|
7608
|
-
|
|
7609
|
-
|
|
7610
|
-
|
|
7611
|
-
|
|
7612
|
-
|
|
7613
|
-
|
|
7614
|
-
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7620
|
-
|
|
7621
|
-
|
|
7622
|
-
|
|
7623
|
-
|
|
7624
|
-
|
|
7625
|
-
|
|
7626
|
-
|
|
7627
|
-
|
|
7628
|
-
|
|
7629
|
-
|
|
7630
|
-
|
|
7631
|
-
|
|
7632
|
-
|
|
7633
|
-
|
|
7634
|
-
|
|
7635
|
-
|
|
7636
|
-
|
|
7637
|
-
|
|
7638
|
-
|
|
7639
|
-
|
|
7640
|
-
observationTokenCount: 0,
|
|
7641
|
-
isObserving: false,
|
|
7642
|
-
isReflecting: false,
|
|
7643
|
-
isBufferingObservation: false,
|
|
7644
|
-
isBufferingReflection: false,
|
|
7645
|
-
lastBufferedAtTokens: 0,
|
|
7646
|
-
lastBufferedAtTime: null,
|
|
7647
|
-
observedTimezone: input.observedTimezone || null,
|
|
7648
|
-
createdAt: now,
|
|
7649
|
-
updatedAt: now
|
|
7765
|
+
try {
|
|
7766
|
+
await collection.insertOne({
|
|
7767
|
+
_id: getGeneration0DocumentId(lookupKey),
|
|
7768
|
+
id,
|
|
7769
|
+
lookupKey,
|
|
7770
|
+
scope: input.scope,
|
|
7771
|
+
resourceId: input.resourceId,
|
|
7772
|
+
threadId: input.threadId || null,
|
|
7773
|
+
activeObservations: "",
|
|
7774
|
+
activeObservationsPendingUpdate: null,
|
|
7775
|
+
originType: "initial",
|
|
7776
|
+
config: input.config,
|
|
7777
|
+
generationCount: 0,
|
|
7778
|
+
lastObservedAt: null,
|
|
7779
|
+
lastReflectionAt: null,
|
|
7780
|
+
pendingMessageTokens: 0,
|
|
7781
|
+
totalTokensObserved: 0,
|
|
7782
|
+
observationTokenCount: 0,
|
|
7783
|
+
isObserving: false,
|
|
7784
|
+
isReflecting: false,
|
|
7785
|
+
isBufferingObservation: false,
|
|
7786
|
+
isBufferingReflection: false,
|
|
7787
|
+
lastBufferedAtTokens: 0,
|
|
7788
|
+
lastBufferedAtTime: null,
|
|
7789
|
+
observedTimezone: input.observedTimezone || null,
|
|
7790
|
+
supersededBy: null,
|
|
7791
|
+
createdAt: now,
|
|
7792
|
+
updatedAt: now
|
|
7793
|
+
});
|
|
7794
|
+
} catch (error) {
|
|
7795
|
+
if (!isDuplicateKeyError(error)) throw error;
|
|
7796
|
+
}
|
|
7797
|
+
const head = await this.#getHeadDoc(collection, lookupKey);
|
|
7798
|
+
if (!head) throw new MastraError({
|
|
7799
|
+
id: createStorageErrorId("MONGODB", "INITIALIZE_OBSERVATIONAL_MEMORY", "NOT_FOUND"),
|
|
7800
|
+
text: `Observational memory record not found after initialization: ${id}`,
|
|
7801
|
+
domain: ErrorDomain.STORAGE,
|
|
7802
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
7803
|
+
details: { id }
|
|
7650
7804
|
});
|
|
7651
|
-
return
|
|
7805
|
+
return this.parseOMDocument(head);
|
|
7652
7806
|
} catch (error) {
|
|
7807
|
+
if (error instanceof MastraError) throw error;
|
|
7653
7808
|
throw new MastraError({
|
|
7654
7809
|
id: createStorageErrorId("MONGODB", "INITIALIZE_OBSERVATIONAL_MEMORY", "FAILED"),
|
|
7655
7810
|
domain: ErrorDomain.STORAGE,
|
|
@@ -7694,6 +7849,7 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
7694
7849
|
lastBufferedAtTime: record.lastBufferedAtTime || null,
|
|
7695
7850
|
observedTimezone: record.observedTimezone || null,
|
|
7696
7851
|
metadata: record.metadata || null,
|
|
7852
|
+
supersededBy: record.supersededBy ?? null,
|
|
7697
7853
|
createdAt: record.createdAt,
|
|
7698
7854
|
updatedAt: record.updatedAt
|
|
7699
7855
|
});
|
|
@@ -7711,28 +7867,40 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
7711
7867
|
}
|
|
7712
7868
|
}
|
|
7713
7869
|
async updateActiveObservations(input) {
|
|
7870
|
+
assertActiveObservationsApplied(await this.commitActiveObservations(input), input.id);
|
|
7871
|
+
}
|
|
7872
|
+
async commitActiveObservations(input) {
|
|
7714
7873
|
try {
|
|
7715
|
-
const now = /* @__PURE__ */ new Date();
|
|
7716
7874
|
const collection = await this.getCollection(OM_TABLE);
|
|
7717
7875
|
const safeTokenCount = Number.isFinite(input.tokenCount) && input.tokenCount >= 0 ? input.tokenCount : 0;
|
|
7718
|
-
|
|
7719
|
-
|
|
7720
|
-
|
|
7721
|
-
|
|
7722
|
-
|
|
7723
|
-
|
|
7724
|
-
|
|
7725
|
-
|
|
7726
|
-
|
|
7727
|
-
|
|
7728
|
-
|
|
7729
|
-
|
|
7730
|
-
|
|
7731
|
-
|
|
7732
|
-
|
|
7733
|
-
|
|
7734
|
-
|
|
7735
|
-
|
|
7876
|
+
for (let attempt = 1; attempt <= OM_MAX_CONDITIONAL_ATTEMPTS; attempt++) {
|
|
7877
|
+
const doc = await collection.findOne({ id: input.id });
|
|
7878
|
+
if (!doc) throw omNotFound("UPDATE_ACTIVE_OBSERVATIONS", input.id);
|
|
7879
|
+
if (doc.supersededBy) return {
|
|
7880
|
+
applied: false,
|
|
7881
|
+
reason: "retired"
|
|
7882
|
+
};
|
|
7883
|
+
if (input.expectedActiveObservations !== void 0 && input.expectedActiveObservations !== (doc.activeObservations || "")) return {
|
|
7884
|
+
applied: false,
|
|
7885
|
+
reason: "conflict"
|
|
7886
|
+
};
|
|
7887
|
+
if ((await collection.updateOne({
|
|
7888
|
+
id: input.id,
|
|
7889
|
+
supersededBy: null,
|
|
7890
|
+
...matchStoredFields(doc, ["activeObservations", "lastObservedAt"])
|
|
7891
|
+
}, {
|
|
7892
|
+
$set: {
|
|
7893
|
+
activeObservations: input.observations,
|
|
7894
|
+
lastObservedAt: maxObservationCursor(doc.lastObservedAt, input.lastObservedAt),
|
|
7895
|
+
pendingMessageTokens: 0,
|
|
7896
|
+
observationTokenCount: safeTokenCount,
|
|
7897
|
+
observedMessageIds: input.observedMessageIds ?? null,
|
|
7898
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
7899
|
+
},
|
|
7900
|
+
$inc: { totalTokensObserved: safeTokenCount }
|
|
7901
|
+
})).matchedCount === 1) return { applied: true };
|
|
7902
|
+
}
|
|
7903
|
+
throw omConflict("UPDATE_ACTIVE_OBSERVATIONS", input.id);
|
|
7736
7904
|
} catch (error) {
|
|
7737
7905
|
if (error instanceof MastraError) throw error;
|
|
7738
7906
|
throw new MastraError({
|
|
@@ -7745,62 +7913,31 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
7745
7913
|
}
|
|
7746
7914
|
async createReflectionGeneration(input) {
|
|
7747
7915
|
try {
|
|
7748
|
-
const
|
|
7749
|
-
|
|
7750
|
-
|
|
7751
|
-
|
|
7752
|
-
|
|
7753
|
-
|
|
7754
|
-
|
|
7755
|
-
|
|
7756
|
-
|
|
7757
|
-
|
|
7758
|
-
|
|
7759
|
-
|
|
7760
|
-
|
|
7761
|
-
|
|
7762
|
-
|
|
7763
|
-
|
|
7764
|
-
|
|
7765
|
-
|
|
7766
|
-
|
|
7767
|
-
|
|
7768
|
-
|
|
7769
|
-
|
|
7770
|
-
lastBufferedAtTime: null,
|
|
7771
|
-
config: input.currentRecord.config,
|
|
7772
|
-
metadata: input.currentRecord.metadata,
|
|
7773
|
-
observedTimezone: input.currentRecord.observedTimezone
|
|
7774
|
-
};
|
|
7775
|
-
await (await this.getCollection(OM_TABLE)).insertOne({
|
|
7776
|
-
id,
|
|
7777
|
-
lookupKey,
|
|
7778
|
-
scope: record.scope,
|
|
7779
|
-
resourceId: record.resourceId,
|
|
7780
|
-
threadId: record.threadId || null,
|
|
7781
|
-
activeObservations: input.reflection,
|
|
7782
|
-
activeObservationsPendingUpdate: null,
|
|
7783
|
-
originType: "reflection",
|
|
7784
|
-
config: record.config,
|
|
7785
|
-
generationCount: input.currentRecord.generationCount + 1,
|
|
7786
|
-
lastObservedAt: record.lastObservedAt || null,
|
|
7787
|
-
lastReflectionAt: now,
|
|
7788
|
-
pendingMessageTokens: record.pendingMessageTokens,
|
|
7789
|
-
totalTokensObserved: record.totalTokensObserved,
|
|
7790
|
-
observationTokenCount: record.observationTokenCount,
|
|
7791
|
-
isObserving: false,
|
|
7792
|
-
isReflecting: false,
|
|
7793
|
-
isBufferingObservation: false,
|
|
7794
|
-
isBufferingReflection: false,
|
|
7795
|
-
lastBufferedAtTokens: 0,
|
|
7796
|
-
lastBufferedAtTime: null,
|
|
7797
|
-
observedTimezone: record.observedTimezone || null,
|
|
7798
|
-
createdAt: now,
|
|
7799
|
-
updatedAt: now,
|
|
7800
|
-
metadata: record.metadata || null
|
|
7916
|
+
const { currentRecord } = input;
|
|
7917
|
+
return await this.#rollOver(currentRecord, (doc) => {
|
|
7918
|
+
const stored = this.parseOMDocument(doc);
|
|
7919
|
+
const plan = planReflectionGenerationText({
|
|
7920
|
+
storedObservations: stored.activeObservations,
|
|
7921
|
+
storedObservationTokenCount: stored.observationTokenCount,
|
|
7922
|
+
snapshotObservations: currentRecord.activeObservations,
|
|
7923
|
+
snapshotObservationTokenCount: currentRecord.observationTokenCount,
|
|
7924
|
+
reflection: input.reflection,
|
|
7925
|
+
tokenCount: input.tokenCount
|
|
7926
|
+
});
|
|
7927
|
+
if (!plan) return null;
|
|
7928
|
+
return {
|
|
7929
|
+
newId: input.newRecordId ?? globalThis.crypto.randomUUID(),
|
|
7930
|
+
mode: plan.mode,
|
|
7931
|
+
reflection: input.reflection,
|
|
7932
|
+
tokenCount: input.tokenCount,
|
|
7933
|
+
snapTextLength: (currentRecord.activeObservations ?? "").length,
|
|
7934
|
+
snapObservationTokenCount: currentRecord.observationTokenCount ?? 0,
|
|
7935
|
+
clearBufferedReflection: false,
|
|
7936
|
+
createdAt: /* @__PURE__ */ new Date()
|
|
7937
|
+
};
|
|
7801
7938
|
});
|
|
7802
|
-
return record;
|
|
7803
7939
|
} catch (error) {
|
|
7940
|
+
if (error instanceof MastraError) throw error;
|
|
7804
7941
|
throw new MastraError({
|
|
7805
7942
|
id: createStorageErrorId("MONGODB", "CREATE_REFLECTION_GENERATION", "FAILED"),
|
|
7806
7943
|
domain: ErrorDomain.STORAGE,
|
|
@@ -7867,23 +8004,12 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
7867
8004
|
}
|
|
7868
8005
|
async setBufferingObservationFlag(id, isBuffering, lastBufferedAtTokens) {
|
|
7869
8006
|
try {
|
|
7870
|
-
const collection = await this.getCollection(OM_TABLE);
|
|
7871
8007
|
const updateDoc = {
|
|
7872
8008
|
isBufferingObservation: isBuffering,
|
|
7873
8009
|
updatedAt: /* @__PURE__ */ new Date()
|
|
7874
8010
|
};
|
|
7875
8011
|
if (lastBufferedAtTokens !== void 0) updateDoc.lastBufferedAtTokens = lastBufferedAtTokens;
|
|
7876
|
-
|
|
7877
|
-
id: createStorageErrorId("MONGODB", "SET_BUFFERING_OBSERVATION_FLAG", "NOT_FOUND"),
|
|
7878
|
-
text: `Observational memory record not found: ${id}`,
|
|
7879
|
-
domain: ErrorDomain.STORAGE,
|
|
7880
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
7881
|
-
details: {
|
|
7882
|
-
id,
|
|
7883
|
-
isBuffering,
|
|
7884
|
-
lastBufferedAtTokens: lastBufferedAtTokens ?? null
|
|
7885
|
-
}
|
|
7886
|
-
});
|
|
8012
|
+
await this.#updateLiveDoc(id, "SET_BUFFERING_OBSERVATION_FLAG", { $set: updateDoc });
|
|
7887
8013
|
} catch (error) {
|
|
7888
8014
|
if (error instanceof MastraError) throw error;
|
|
7889
8015
|
throw new MastraError({
|
|
@@ -7954,19 +8080,10 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
7954
8080
|
}
|
|
7955
8081
|
});
|
|
7956
8082
|
try {
|
|
7957
|
-
|
|
8083
|
+
await this.#updateLiveDoc(id, "SET_PENDING_MESSAGE_TOKENS", { $set: {
|
|
7958
8084
|
pendingMessageTokens: tokenCount,
|
|
7959
8085
|
updatedAt: /* @__PURE__ */ new Date()
|
|
7960
|
-
} })
|
|
7961
|
-
id: createStorageErrorId("MONGODB", "SET_PENDING_MESSAGE_TOKENS", "NOT_FOUND"),
|
|
7962
|
-
text: `Observational memory record not found: ${id}`,
|
|
7963
|
-
domain: ErrorDomain.STORAGE,
|
|
7964
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
7965
|
-
details: {
|
|
7966
|
-
id,
|
|
7967
|
-
tokenCount
|
|
7968
|
-
}
|
|
7969
|
-
});
|
|
8086
|
+
} });
|
|
7970
8087
|
} catch (error) {
|
|
7971
8088
|
if (error instanceof MastraError) throw error;
|
|
7972
8089
|
throw new MastraError({
|
|
@@ -8008,6 +8125,9 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
8008
8125
|
}
|
|
8009
8126
|
}
|
|
8010
8127
|
async updateBufferedObservations(input) {
|
|
8128
|
+
await this.appendBufferedObservations(input);
|
|
8129
|
+
}
|
|
8130
|
+
async appendBufferedObservations(input) {
|
|
8011
8131
|
try {
|
|
8012
8132
|
const collection = await this.getCollection(OM_TABLE);
|
|
8013
8133
|
const newChunk = {
|
|
@@ -8025,18 +8145,65 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
8025
8145
|
extractedValues: input.chunk.extractedValues,
|
|
8026
8146
|
extractionFailures: input.chunk.extractionFailures
|
|
8027
8147
|
};
|
|
8028
|
-
const
|
|
8029
|
-
|
|
8030
|
-
|
|
8031
|
-
|
|
8032
|
-
|
|
8033
|
-
|
|
8034
|
-
|
|
8035
|
-
|
|
8036
|
-
|
|
8037
|
-
|
|
8038
|
-
|
|
8039
|
-
|
|
8148
|
+
const notCovered = { $or: [{ lastObservedAt: null }, { lastObservedAt: { $lt: /* @__PURE__ */ new Date(new Date(input.chunk.lastObservedAt).getTime() - 1) } }] };
|
|
8149
|
+
const lastBufferedAtTime = input.lastBufferedAtTime ? new Date(input.lastBufferedAtTime) : void 0;
|
|
8150
|
+
let targetId = input.id;
|
|
8151
|
+
let hops = 0;
|
|
8152
|
+
let retries = 0;
|
|
8153
|
+
for (;;) {
|
|
8154
|
+
if ((await collection.updateOne({
|
|
8155
|
+
id: targetId,
|
|
8156
|
+
supersededBy: null,
|
|
8157
|
+
"bufferedObservationChunks.cycleId": { $ne: input.chunk.cycleId },
|
|
8158
|
+
...notCovered,
|
|
8159
|
+
$and: [{ $or: [
|
|
8160
|
+
{ bufferedObservationChunks: { $exists: false } },
|
|
8161
|
+
{ bufferedObservationChunks: null },
|
|
8162
|
+
{ bufferedObservationChunks: { $type: "array" } }
|
|
8163
|
+
] }]
|
|
8164
|
+
}, [{ $set: {
|
|
8165
|
+
bufferedObservationChunks: { $concatArrays: [{ $ifNull: ["$bufferedObservationChunks", []] }, { $literal: [newChunk] }] },
|
|
8166
|
+
...lastBufferedAtTime ? { lastBufferedAtTime: { $max: ["$lastBufferedAtTime", { $literal: lastBufferedAtTime }] } } : {},
|
|
8167
|
+
updatedAt: { $literal: /* @__PURE__ */ new Date() }
|
|
8168
|
+
} }])).matchedCount === 1) return {
|
|
8169
|
+
persisted: true,
|
|
8170
|
+
recordId: targetId
|
|
8171
|
+
};
|
|
8172
|
+
const doc = await collection.findOne({ id: targetId });
|
|
8173
|
+
if (!doc) throw omNotFound("UPDATE_BUFFERED_OBSERVATIONS", targetId);
|
|
8174
|
+
if (doc.supersededBy) {
|
|
8175
|
+
const head = hops < OM_MAX_HEAD_HOPS ? await this.#getHeadDoc(collection, doc.lookupKey) : null;
|
|
8176
|
+
if (!head || head.supersededBy || head.id === targetId) throw omNoLiveHead(input.id);
|
|
8177
|
+
targetId = head.id;
|
|
8178
|
+
hops++;
|
|
8179
|
+
continue;
|
|
8180
|
+
}
|
|
8181
|
+
const stored = doc.bufferedObservationChunks;
|
|
8182
|
+
if (Array.isArray(stored) && stored.some((c) => c?.cycleId === input.chunk.cycleId)) return {
|
|
8183
|
+
persisted: false,
|
|
8184
|
+
recordId: targetId
|
|
8185
|
+
};
|
|
8186
|
+
if (isBufferedChunkCoveredByCursor(input.chunk.lastObservedAt, doc.lastObservedAt)) return {
|
|
8187
|
+
persisted: false,
|
|
8188
|
+
recordId: targetId
|
|
8189
|
+
};
|
|
8190
|
+
if (stored !== void 0 && stored !== null && !Array.isArray(stored)) {
|
|
8191
|
+
if ((await collection.updateOne({
|
|
8192
|
+
id: targetId,
|
|
8193
|
+
supersededBy: null,
|
|
8194
|
+
bufferedObservationChunks: stored,
|
|
8195
|
+
...notCovered
|
|
8196
|
+
}, { $set: {
|
|
8197
|
+
bufferedObservationChunks: [newChunk],
|
|
8198
|
+
...lastBufferedAtTime ? { lastBufferedAtTime: maxObservationCursor(doc.lastBufferedAtTime, lastBufferedAtTime) } : {},
|
|
8199
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
8200
|
+
} })).matchedCount === 1) return {
|
|
8201
|
+
persisted: true,
|
|
8202
|
+
recordId: targetId
|
|
8203
|
+
};
|
|
8204
|
+
}
|
|
8205
|
+
if (++retries >= OM_MAX_CONDITIONAL_ATTEMPTS) throw omConflict("UPDATE_BUFFERED_OBSERVATIONS", targetId);
|
|
8206
|
+
}
|
|
8040
8207
|
} catch (error) {
|
|
8041
8208
|
if (error instanceof MastraError) throw error;
|
|
8042
8209
|
throw new MastraError({
|
|
@@ -8050,16 +8217,7 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
8050
8217
|
async swapBufferedToActive(input) {
|
|
8051
8218
|
try {
|
|
8052
8219
|
const collection = await this.getCollection(OM_TABLE);
|
|
8053
|
-
const
|
|
8054
|
-
if (!doc) throw new MastraError({
|
|
8055
|
-
id: createStorageErrorId("MONGODB", "SWAP_BUFFERED_TO_ACTIVE", "NOT_FOUND"),
|
|
8056
|
-
text: `Observational memory record not found: ${input.id}`,
|
|
8057
|
-
domain: ErrorDomain.STORAGE,
|
|
8058
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
8059
|
-
details: { id: input.id }
|
|
8060
|
-
});
|
|
8061
|
-
const chunks = Array.isArray(doc.bufferedObservationChunks) ? doc.bufferedObservationChunks : [];
|
|
8062
|
-
if (chunks.length === 0) return {
|
|
8220
|
+
const emptyResult = {
|
|
8063
8221
|
chunksActivated: 0,
|
|
8064
8222
|
messageTokensActivated: 0,
|
|
8065
8223
|
observationTokensActivated: 0,
|
|
@@ -8067,95 +8225,107 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
8067
8225
|
activatedCycleIds: [],
|
|
8068
8226
|
activatedMessageIds: []
|
|
8069
8227
|
};
|
|
8070
|
-
|
|
8071
|
-
|
|
8072
|
-
|
|
8073
|
-
|
|
8074
|
-
|
|
8075
|
-
|
|
8076
|
-
|
|
8077
|
-
|
|
8078
|
-
|
|
8079
|
-
const
|
|
8080
|
-
|
|
8081
|
-
|
|
8082
|
-
|
|
8083
|
-
|
|
8228
|
+
for (let attempt = 1; attempt <= OM_MAX_CONDITIONAL_ATTEMPTS; attempt++) {
|
|
8229
|
+
const doc = await collection.findOne({ id: input.id });
|
|
8230
|
+
if (!doc) throw omNotFound("SWAP_BUFFERED_TO_ACTIVE", input.id);
|
|
8231
|
+
if (doc.supersededBy) return {
|
|
8232
|
+
...emptyResult,
|
|
8233
|
+
retired: true
|
|
8234
|
+
};
|
|
8235
|
+
const persistedChunks = Array.isArray(doc.bufferedObservationChunks) ? doc.bufferedObservationChunks : [];
|
|
8236
|
+
const refreshedWeights = new Map((Array.isArray(input.bufferedChunks) ? input.bufferedChunks : []).map((c) => [c.id, c.messageTokens]));
|
|
8237
|
+
const chunks = persistedChunks.map((c) => refreshedWeights.has(c.id) ? {
|
|
8238
|
+
...c,
|
|
8239
|
+
messageTokens: refreshedWeights.get(c.id)
|
|
8240
|
+
} : c);
|
|
8241
|
+
if (chunks.length === 0) return emptyResult;
|
|
8242
|
+
const retentionFloor = input.messageTokensThreshold * (1 - input.activationRatio);
|
|
8243
|
+
const targetMessageTokens = Math.max(0, input.currentPendingTokens - retentionFloor);
|
|
8244
|
+
let cumulativeMessageTokens = 0;
|
|
8245
|
+
let bestOverBoundary = 0;
|
|
8246
|
+
let bestOverTokens = 0;
|
|
8247
|
+
let bestUnderBoundary = 0;
|
|
8248
|
+
let bestUnderTokens = 0;
|
|
8249
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
8250
|
+
cumulativeMessageTokens += chunks[i].messageTokens ?? 0;
|
|
8251
|
+
const boundary = i + 1;
|
|
8252
|
+
if (cumulativeMessageTokens >= targetMessageTokens) {
|
|
8253
|
+
if (bestOverBoundary === 0 || cumulativeMessageTokens < bestOverTokens) {
|
|
8254
|
+
bestOverBoundary = boundary;
|
|
8255
|
+
bestOverTokens = cumulativeMessageTokens;
|
|
8256
|
+
}
|
|
8257
|
+
} else if (cumulativeMessageTokens > bestUnderTokens) {
|
|
8258
|
+
bestUnderBoundary = boundary;
|
|
8259
|
+
bestUnderTokens = cumulativeMessageTokens;
|
|
8084
8260
|
}
|
|
8085
|
-
} else if (cumulativeMessageTokens > bestUnderTokens) {
|
|
8086
|
-
bestUnderBoundary = boundary;
|
|
8087
|
-
bestUnderTokens = cumulativeMessageTokens;
|
|
8088
8261
|
}
|
|
8262
|
+
const maxOvershoot = retentionFloor * .95;
|
|
8263
|
+
const overshoot = bestOverTokens - targetMessageTokens;
|
|
8264
|
+
const remainingAfterOver = input.currentPendingTokens - bestOverTokens;
|
|
8265
|
+
const remainingAfterUnder = input.currentPendingTokens - bestUnderTokens;
|
|
8266
|
+
const minRemaining = Math.min(1e3, retentionFloor);
|
|
8267
|
+
let chunksToActivate;
|
|
8268
|
+
if (input.forceMaxActivation && bestOverBoundary > 0 && remainingAfterOver >= minRemaining) chunksToActivate = bestOverBoundary;
|
|
8269
|
+
else if (bestOverBoundary > 0 && overshoot <= maxOvershoot && remainingAfterOver >= minRemaining) chunksToActivate = bestOverBoundary;
|
|
8270
|
+
else if (bestUnderBoundary > 0 && remainingAfterUnder >= minRemaining) chunksToActivate = bestUnderBoundary;
|
|
8271
|
+
else if (bestOverBoundary > 0) chunksToActivate = bestOverBoundary;
|
|
8272
|
+
else chunksToActivate = 1;
|
|
8273
|
+
const activatedChunks = chunks.slice(0, chunksToActivate);
|
|
8274
|
+
const remainingChunks = persistedChunks.slice(chunksToActivate);
|
|
8275
|
+
const activatedContent = activatedChunks.map((c) => c.observations).join("\n\n");
|
|
8276
|
+
const activatedTokens = activatedChunks.reduce((sum, c) => sum + c.tokenCount, 0);
|
|
8277
|
+
const activatedMessageTokens = activatedChunks.reduce((sum, c) => sum + (c.messageTokens ?? 0), 0);
|
|
8278
|
+
const activatedMessageCount = activatedChunks.reduce((sum, c) => sum + c.messageIds.length, 0);
|
|
8279
|
+
const activatedCycleIds = activatedChunks.map((c) => c.cycleId).filter((id) => !!id);
|
|
8280
|
+
const activatedMessageIds = activatedChunks.flatMap((c) => c.messageIds ?? []);
|
|
8281
|
+
const latestChunk = activatedChunks[activatedChunks.length - 1];
|
|
8282
|
+
const lastObservedAt = input.lastObservedAt ?? (latestChunk?.lastObservedAt ? new Date(latestChunk.lastObservedAt) : /* @__PURE__ */ new Date());
|
|
8283
|
+
const existingActive = doc.activeObservations || "";
|
|
8284
|
+
const existingTokenCount = Number(doc.observationTokenCount || 0);
|
|
8285
|
+
const boundary = `\n\n--- message boundary (${lastObservedAt.toISOString()}) ---\n\n`;
|
|
8286
|
+
const newActive = existingActive ? `${existingActive}${boundary}${activatedContent}` : activatedContent;
|
|
8287
|
+
const newTokenCount = existingTokenCount + activatedTokens;
|
|
8288
|
+
const existingPending = Number(doc.pendingMessageTokens || 0);
|
|
8289
|
+
const newPending = Math.max(0, existingPending - activatedMessageTokens);
|
|
8290
|
+
if ((await collection.updateOne({
|
|
8291
|
+
id: input.id,
|
|
8292
|
+
supersededBy: null,
|
|
8293
|
+
...matchStoredFields(doc, [
|
|
8294
|
+
"bufferedObservationChunks",
|
|
8295
|
+
"activeObservations",
|
|
8296
|
+
"observationTokenCount",
|
|
8297
|
+
"pendingMessageTokens",
|
|
8298
|
+
"lastObservedAt"
|
|
8299
|
+
])
|
|
8300
|
+
}, { $set: {
|
|
8301
|
+
activeObservations: newActive,
|
|
8302
|
+
observationTokenCount: newTokenCount,
|
|
8303
|
+
pendingMessageTokens: newPending,
|
|
8304
|
+
bufferedObservationChunks: remainingChunks,
|
|
8305
|
+
lastObservedAt: maxObservationCursor(doc.lastObservedAt, lastObservedAt),
|
|
8306
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
8307
|
+
} })).matchedCount === 0) continue;
|
|
8308
|
+
const latestChunkHints = activatedChunks[activatedChunks.length - 1];
|
|
8309
|
+
return {
|
|
8310
|
+
chunksActivated: activatedChunks.length,
|
|
8311
|
+
messageTokensActivated: activatedMessageTokens,
|
|
8312
|
+
observationTokensActivated: activatedTokens,
|
|
8313
|
+
messagesActivated: activatedMessageCount,
|
|
8314
|
+
activatedCycleIds,
|
|
8315
|
+
activatedMessageIds,
|
|
8316
|
+
observations: activatedContent,
|
|
8317
|
+
perChunk: activatedChunks.map((c) => ({
|
|
8318
|
+
cycleId: c.cycleId ?? "",
|
|
8319
|
+
messageTokens: c.messageTokens ?? 0,
|
|
8320
|
+
observationTokens: c.tokenCount,
|
|
8321
|
+
messageCount: c.messageIds.length,
|
|
8322
|
+
observations: c.observations
|
|
8323
|
+
})),
|
|
8324
|
+
suggestedContinuation: latestChunkHints?.suggestedContinuation ?? void 0,
|
|
8325
|
+
currentTask: latestChunkHints?.currentTask ?? void 0
|
|
8326
|
+
};
|
|
8089
8327
|
}
|
|
8090
|
-
|
|
8091
|
-
const overshoot = bestOverTokens - targetMessageTokens;
|
|
8092
|
-
const remainingAfterOver = input.currentPendingTokens - bestOverTokens;
|
|
8093
|
-
const remainingAfterUnder = input.currentPendingTokens - bestUnderTokens;
|
|
8094
|
-
const minRemaining = Math.min(1e3, retentionFloor);
|
|
8095
|
-
let chunksToActivate;
|
|
8096
|
-
if (input.forceMaxActivation && bestOverBoundary > 0 && remainingAfterOver >= minRemaining) chunksToActivate = bestOverBoundary;
|
|
8097
|
-
else if (bestOverBoundary > 0 && overshoot <= maxOvershoot && remainingAfterOver >= minRemaining) chunksToActivate = bestOverBoundary;
|
|
8098
|
-
else if (bestUnderBoundary > 0 && remainingAfterUnder >= minRemaining) chunksToActivate = bestUnderBoundary;
|
|
8099
|
-
else if (bestOverBoundary > 0) chunksToActivate = bestOverBoundary;
|
|
8100
|
-
else chunksToActivate = 1;
|
|
8101
|
-
const activatedChunks = chunks.slice(0, chunksToActivate);
|
|
8102
|
-
const remainingChunks = chunks.slice(chunksToActivate);
|
|
8103
|
-
const activatedContent = activatedChunks.map((c) => c.observations).join("\n\n");
|
|
8104
|
-
const activatedTokens = activatedChunks.reduce((sum, c) => sum + c.tokenCount, 0);
|
|
8105
|
-
const activatedMessageTokens = activatedChunks.reduce((sum, c) => sum + (c.messageTokens ?? 0), 0);
|
|
8106
|
-
const activatedMessageCount = activatedChunks.reduce((sum, c) => sum + c.messageIds.length, 0);
|
|
8107
|
-
const activatedCycleIds = activatedChunks.map((c) => c.cycleId).filter((id) => !!id);
|
|
8108
|
-
const activatedMessageIds = activatedChunks.flatMap((c) => c.messageIds ?? []);
|
|
8109
|
-
const latestChunk = activatedChunks[activatedChunks.length - 1];
|
|
8110
|
-
const lastObservedAt = input.lastObservedAt ?? (latestChunk?.lastObservedAt ? new Date(latestChunk.lastObservedAt) : /* @__PURE__ */ new Date());
|
|
8111
|
-
const existingActive = doc.activeObservations || "";
|
|
8112
|
-
const existingTokenCount = Number(doc.observationTokenCount || 0);
|
|
8113
|
-
const boundary = `\n\n--- message boundary (${lastObservedAt.toISOString()}) ---\n\n`;
|
|
8114
|
-
const newActive = existingActive ? `${existingActive}${boundary}${activatedContent}` : activatedContent;
|
|
8115
|
-
const newTokenCount = existingTokenCount + activatedTokens;
|
|
8116
|
-
const existingPending = Number(doc.pendingMessageTokens || 0);
|
|
8117
|
-
const newPending = Math.max(0, existingPending - activatedMessageTokens);
|
|
8118
|
-
if ((await collection.updateOne({
|
|
8119
|
-
id: input.id,
|
|
8120
|
-
bufferedObservationChunks: {
|
|
8121
|
-
$exists: true,
|
|
8122
|
-
$ne: null,
|
|
8123
|
-
$not: { $size: 0 }
|
|
8124
|
-
}
|
|
8125
|
-
}, { $set: {
|
|
8126
|
-
activeObservations: newActive,
|
|
8127
|
-
observationTokenCount: newTokenCount,
|
|
8128
|
-
pendingMessageTokens: newPending,
|
|
8129
|
-
bufferedObservationChunks: remainingChunks,
|
|
8130
|
-
lastObservedAt,
|
|
8131
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
8132
|
-
} })).modifiedCount === 0) return {
|
|
8133
|
-
chunksActivated: 0,
|
|
8134
|
-
messageTokensActivated: 0,
|
|
8135
|
-
observationTokensActivated: 0,
|
|
8136
|
-
messagesActivated: 0,
|
|
8137
|
-
activatedCycleIds: [],
|
|
8138
|
-
activatedMessageIds: []
|
|
8139
|
-
};
|
|
8140
|
-
const latestChunkHints = activatedChunks[activatedChunks.length - 1];
|
|
8141
|
-
return {
|
|
8142
|
-
chunksActivated: activatedChunks.length,
|
|
8143
|
-
messageTokensActivated: activatedMessageTokens,
|
|
8144
|
-
observationTokensActivated: activatedTokens,
|
|
8145
|
-
messagesActivated: activatedMessageCount,
|
|
8146
|
-
activatedCycleIds,
|
|
8147
|
-
activatedMessageIds,
|
|
8148
|
-
observations: activatedContent,
|
|
8149
|
-
perChunk: activatedChunks.map((c) => ({
|
|
8150
|
-
cycleId: c.cycleId ?? "",
|
|
8151
|
-
messageTokens: c.messageTokens ?? 0,
|
|
8152
|
-
observationTokens: c.tokenCount,
|
|
8153
|
-
messageCount: c.messageIds.length,
|
|
8154
|
-
observations: c.observations
|
|
8155
|
-
})),
|
|
8156
|
-
suggestedContinuation: latestChunkHints?.suggestedContinuation ?? void 0,
|
|
8157
|
-
currentTask: latestChunkHints?.currentTask ?? void 0
|
|
8158
|
-
};
|
|
8328
|
+
throw omConflict("SWAP_BUFFERED_TO_ACTIVE", input.id);
|
|
8159
8329
|
} catch (error) {
|
|
8160
8330
|
if (error instanceof MastraError) throw error;
|
|
8161
8331
|
throw new MastraError({
|
|
@@ -8208,39 +8378,33 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
8208
8378
|
}
|
|
8209
8379
|
async swapBufferedReflectionToActive(input) {
|
|
8210
8380
|
try {
|
|
8211
|
-
const
|
|
8212
|
-
|
|
8213
|
-
|
|
8214
|
-
|
|
8215
|
-
|
|
8216
|
-
|
|
8217
|
-
|
|
8218
|
-
|
|
8219
|
-
|
|
8220
|
-
|
|
8221
|
-
|
|
8222
|
-
|
|
8223
|
-
|
|
8224
|
-
|
|
8225
|
-
|
|
8226
|
-
|
|
8227
|
-
|
|
8228
|
-
|
|
8229
|
-
|
|
8230
|
-
|
|
8231
|
-
|
|
8232
|
-
|
|
8233
|
-
|
|
8234
|
-
|
|
8381
|
+
const { currentRecord } = input;
|
|
8382
|
+
return await this.#rollOver(currentRecord, (doc) => {
|
|
8383
|
+
const stored = this.parseOMDocument(doc);
|
|
8384
|
+
const bufferedReflection = stored.bufferedReflection || "";
|
|
8385
|
+
if (!bufferedReflection) throw new MastraError({
|
|
8386
|
+
id: createStorageErrorId("MONGODB", "SWAP_BUFFERED_REFLECTION_TO_ACTIVE", "NO_CONTENT"),
|
|
8387
|
+
text: "No buffered reflection to swap",
|
|
8388
|
+
domain: ErrorDomain.STORAGE,
|
|
8389
|
+
category: ErrorCategory.USER,
|
|
8390
|
+
details: { id: currentRecord.id }
|
|
8391
|
+
});
|
|
8392
|
+
if (!isAppendOnlySince(stored.activeObservations, currentRecord.activeObservations)) return null;
|
|
8393
|
+
const reflectedLineCount = stored.reflectedObservationLineCount ?? 0;
|
|
8394
|
+
const unreflectedContent = (stored.activeObservations || "").split("\n").slice(reflectedLineCount).join("\n").trim();
|
|
8395
|
+
const newObservations = unreflectedContent ? `${bufferedReflection}\n\n${unreflectedContent}` : bufferedReflection;
|
|
8396
|
+
const tokenCount = input.tokenCount + Math.max(0, (stored.observationTokenCount ?? 0) - (currentRecord.observationTokenCount ?? 0));
|
|
8397
|
+
return {
|
|
8398
|
+
newId: input.newRecordId ?? globalThis.crypto.randomUUID(),
|
|
8399
|
+
mode: "equal",
|
|
8400
|
+
reflection: newObservations,
|
|
8401
|
+
tokenCount,
|
|
8402
|
+
snapTextLength: (stored.activeObservations ?? "").length,
|
|
8403
|
+
snapObservationTokenCount: stored.observationTokenCount ?? 0,
|
|
8404
|
+
clearBufferedReflection: true,
|
|
8405
|
+
createdAt: /* @__PURE__ */ new Date()
|
|
8406
|
+
};
|
|
8235
8407
|
});
|
|
8236
|
-
await collection.updateOne({ id: input.currentRecord.id }, { $set: {
|
|
8237
|
-
bufferedReflection: null,
|
|
8238
|
-
bufferedReflectionTokens: null,
|
|
8239
|
-
bufferedReflectionInputTokens: null,
|
|
8240
|
-
reflectedObservationLineCount: null,
|
|
8241
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
8242
|
-
} });
|
|
8243
|
-
return newRecord;
|
|
8244
8408
|
} catch (error) {
|
|
8245
8409
|
if (error instanceof MastraError) throw error;
|
|
8246
8410
|
throw new MastraError({
|
|
@@ -8251,6 +8415,168 @@ var MemoryStorageMongoDB = class MemoryStorageMongoDB extends MemoryStorage {
|
|
|
8251
8415
|
}, error);
|
|
8252
8416
|
}
|
|
8253
8417
|
}
|
|
8418
|
+
/**
|
|
8419
|
+
* The head of a lookup key by the canonical order (`generationCount DESC, createdAt ASC, id ASC`).
|
|
8420
|
+
* A head that was fenced for rollover but whose successor insert never ran (crash, or still in
|
|
8421
|
+
* flight in another process) is rolled forward here, so every head read reaches the successor.
|
|
8422
|
+
*/
|
|
8423
|
+
async #getHeadDoc(collection, lookupKey) {
|
|
8424
|
+
for (let hop = 0; hop <= OM_MAX_HEAD_HOPS; hop++) {
|
|
8425
|
+
const head = await collection.findOne({ lookupKey }, { sort: OM_HEAD_SORT });
|
|
8426
|
+
if (!head || !head.supersededBy || !head.pendingSuccessor) return head;
|
|
8427
|
+
await this.#completeRollover(collection, head, head.pendingSuccessor);
|
|
8428
|
+
}
|
|
8429
|
+
throw omNoLiveHead(lookupKey);
|
|
8430
|
+
}
|
|
8431
|
+
/**
|
|
8432
|
+
* A single-document update aimed at a live record. A retired (or concurrently fenced) id is
|
|
8433
|
+
* redirected to the head, at most `OM_MAX_HEAD_HOPS` times.
|
|
8434
|
+
*/
|
|
8435
|
+
async #updateLiveDoc(id, operation, update) {
|
|
8436
|
+
const collection = await this.getCollection(OM_TABLE);
|
|
8437
|
+
let targetId = id;
|
|
8438
|
+
for (let hop = 0; hop <= OM_MAX_HEAD_HOPS; hop++) {
|
|
8439
|
+
if ((await collection.updateOne({
|
|
8440
|
+
id: targetId,
|
|
8441
|
+
supersededBy: null
|
|
8442
|
+
}, update)).matchedCount === 1) return targetId;
|
|
8443
|
+
const doc = await collection.findOne({ id: targetId }, { projection: {
|
|
8444
|
+
lookupKey: 1,
|
|
8445
|
+
supersededBy: 1
|
|
8446
|
+
} });
|
|
8447
|
+
if (!doc) throw omNotFound(operation, targetId);
|
|
8448
|
+
if (!doc.supersededBy) continue;
|
|
8449
|
+
const head = await this.#getHeadDoc(collection, doc.lookupKey);
|
|
8450
|
+
if (!head || head.supersededBy || head.id === targetId) throw omNoLiveHead(id);
|
|
8451
|
+
targetId = head.id;
|
|
8452
|
+
}
|
|
8453
|
+
throw omNoLiveHead(id);
|
|
8454
|
+
}
|
|
8455
|
+
/**
|
|
8456
|
+
* Roll the live record `currentRecord.id` over to a new generation.
|
|
8457
|
+
*
|
|
8458
|
+
* 1. Read the stored document and decide (`decide` returns the successor inputs, or `null`
|
|
8459
|
+
* when the snapshot is stale).
|
|
8460
|
+
* 2. Fence: one conditional update sets `supersededBy` and the small `pendingSuccessor`
|
|
8461
|
+
* payload, matching only if the fields the decision read are unchanged. From here the old
|
|
8462
|
+
* document is frozen — no lifecycle write matches it — so its pre-image is its final state.
|
|
8463
|
+
* 3. Insert the successor built purely from that pre-image, then clear the old document's
|
|
8464
|
+
* chunks (they moved) and `pendingSuccessor`.
|
|
8465
|
+
*
|
|
8466
|
+
* A crash between 2 and 3 is finished by the next head read (`#getHeadDoc`) or startup.
|
|
8467
|
+
*/
|
|
8468
|
+
async #rollOver(currentRecord, decide) {
|
|
8469
|
+
const collection = await this.getCollection(OM_TABLE);
|
|
8470
|
+
for (let attempt = 1; attempt <= OM_MAX_CONDITIONAL_ATTEMPTS; attempt++) {
|
|
8471
|
+
const doc = await collection.findOne({ id: currentRecord.id });
|
|
8472
|
+
if (!doc) return currentRecord;
|
|
8473
|
+
if (doc.supersededBy) {
|
|
8474
|
+
const head = await this.#getHeadDoc(collection, doc.lookupKey);
|
|
8475
|
+
return head ? this.parseOMDocument(head) : currentRecord;
|
|
8476
|
+
}
|
|
8477
|
+
const pending = decide(doc);
|
|
8478
|
+
if (!pending) return this.parseOMDocument(doc);
|
|
8479
|
+
const frozen = await collection.findOneAndUpdate({
|
|
8480
|
+
id: doc.id,
|
|
8481
|
+
supersededBy: null,
|
|
8482
|
+
...matchStoredFields(doc, [
|
|
8483
|
+
"activeObservations",
|
|
8484
|
+
"observationTokenCount",
|
|
8485
|
+
"bufferedReflection",
|
|
8486
|
+
"reflectedObservationLineCount"
|
|
8487
|
+
])
|
|
8488
|
+
}, { $set: {
|
|
8489
|
+
supersededBy: pending.newId,
|
|
8490
|
+
pendingSuccessor: pending
|
|
8491
|
+
} }, { returnDocument: "before" });
|
|
8492
|
+
if (!frozen) continue;
|
|
8493
|
+
return this.parseOMDocument(await this.#completeRollover(collection, frozen, pending));
|
|
8494
|
+
}
|
|
8495
|
+
throw omConflict("CREATE_REFLECTION_GENERATION", currentRecord.id);
|
|
8496
|
+
}
|
|
8497
|
+
/** Insert the successor of a fenced document (idempotent) and clean up the old document. */
|
|
8498
|
+
async #completeRollover(collection, frozen, pending) {
|
|
8499
|
+
try {
|
|
8500
|
+
await collection.insertOne(buildSuccessorDocument(frozen, pending));
|
|
8501
|
+
} catch (error) {
|
|
8502
|
+
if (!isDuplicateKeyError(error)) throw error;
|
|
8503
|
+
}
|
|
8504
|
+
await collection.updateOne({
|
|
8505
|
+
id: frozen.id,
|
|
8506
|
+
supersededBy: pending.newId
|
|
8507
|
+
}, {
|
|
8508
|
+
$set: {
|
|
8509
|
+
bufferedObservationChunks: [],
|
|
8510
|
+
...pending.clearBufferedReflection ? {
|
|
8511
|
+
bufferedReflection: null,
|
|
8512
|
+
bufferedReflectionTokens: null,
|
|
8513
|
+
bufferedReflectionInputTokens: null,
|
|
8514
|
+
reflectedObservationLineCount: null
|
|
8515
|
+
} : {},
|
|
8516
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
8517
|
+
},
|
|
8518
|
+
$unset: { pendingSuccessor: "" }
|
|
8519
|
+
});
|
|
8520
|
+
const successor = await collection.findOne({ id: pending.newId });
|
|
8521
|
+
if (!successor) throw omNotFound("CREATE_REFLECTION_GENERATION", pending.newId);
|
|
8522
|
+
return successor;
|
|
8523
|
+
}
|
|
8524
|
+
/**
|
|
8525
|
+
* Startup maintenance for the `supersededBy` marker:
|
|
8526
|
+
* - finish rollovers interrupted after the fence (insert the successor if missing, clean up);
|
|
8527
|
+
* - for keys with more than one live document (left by older adapter versions, which never set
|
|
8528
|
+
* `supersededBy`), retire every live document that sorts after the canonical head. The head's
|
|
8529
|
+
* ordering key is embedded in the filter, so a document created by a concurrent rollover
|
|
8530
|
+
* (a newer generation) never matches.
|
|
8531
|
+
*/
|
|
8532
|
+
async #maintainSupersededBy() {
|
|
8533
|
+
const collection = await this.getCollection(OM_TABLE);
|
|
8534
|
+
const interrupted = await collection.find({
|
|
8535
|
+
supersededBy: { $nin: [null] },
|
|
8536
|
+
pendingSuccessor: { $exists: true }
|
|
8537
|
+
}).toArray();
|
|
8538
|
+
for (const doc of interrupted) await this.#completeRollover(collection, doc, doc.pendingSuccessor);
|
|
8539
|
+
const keys = await collection.aggregate([
|
|
8540
|
+
{ $match: { supersededBy: null } },
|
|
8541
|
+
{ $group: {
|
|
8542
|
+
_id: "$lookupKey",
|
|
8543
|
+
live: { $sum: 1 }
|
|
8544
|
+
} },
|
|
8545
|
+
{ $match: { live: { $gt: 1 } } }
|
|
8546
|
+
]).toArray();
|
|
8547
|
+
for (const { _id: lookupKey } of keys) {
|
|
8548
|
+
const head = await this.#getHeadDoc(collection, lookupKey);
|
|
8549
|
+
if (!head || head.supersededBy) continue;
|
|
8550
|
+
await collection.updateMany({
|
|
8551
|
+
lookupKey,
|
|
8552
|
+
supersededBy: null,
|
|
8553
|
+
$or: [
|
|
8554
|
+
{ generationCount: { $lt: head.generationCount } },
|
|
8555
|
+
{
|
|
8556
|
+
generationCount: head.generationCount,
|
|
8557
|
+
createdAt: { $gt: head.createdAt }
|
|
8558
|
+
},
|
|
8559
|
+
{
|
|
8560
|
+
generationCount: head.generationCount,
|
|
8561
|
+
createdAt: head.createdAt,
|
|
8562
|
+
id: { $gt: head.id }
|
|
8563
|
+
}
|
|
8564
|
+
]
|
|
8565
|
+
}, { $set: { supersededBy: head.id } });
|
|
8566
|
+
}
|
|
8567
|
+
}
|
|
8568
|
+
/**
|
|
8569
|
+
* Cross-process rollover and initialization rely on the unique `id` index (a duplicate insert
|
|
8570
|
+
* of the same successor or generation-0 id must fail). Warn once if it is missing, e.g. under
|
|
8571
|
+
* `skipDefaultIndexes`.
|
|
8572
|
+
*/
|
|
8573
|
+
async #warnIfOMIdIndexMissing() {
|
|
8574
|
+
try {
|
|
8575
|
+
if (!(await (await this.getCollection(OM_TABLE)).listIndexes().toArray().catch((error) => error?.code === 26 ? [] : Promise.reject(error))).some((index) => index.unique && Object.keys(index.key ?? {}).length === 1 && index.key.id === 1)) this.logger?.warn?.(`MongoDB collection "${OM_TABLE}" has no unique index on { id: 1 }. Observational memory needs it to stay consistent when several processes write the same thread or resource. Create it with db.${OM_TABLE}.createIndex({ id: 1 }, { unique: true }).`);
|
|
8576
|
+
} catch (error) {
|
|
8577
|
+
this.logger?.warn?.(`Could not list indexes on MongoDB collection "${OM_TABLE}":`, error);
|
|
8578
|
+
}
|
|
8579
|
+
}
|
|
8254
8580
|
};
|
|
8255
8581
|
//#endregion
|
|
8256
8582
|
//#region src/storage/domains/notifications/index.ts
|