@mastra/mysql 0.13.0-alpha.1 → 0.13.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_types/@internal_core/dist/memory/index.d.ts +102 -0
- package/dist/_types/@internal_core/package.json +1 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/reference-storage-retention.md +21 -18
- package/dist/index.cjs +544 -346
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +544 -346
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/memory/index.d.ts +36 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/package.json +6 -5
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@ import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
|
|
|
2
2
|
import { AGENTS_SCHEMA, AGENT_VERSIONS_SCHEMA, AgentsStorage, BackgroundTasksStorage, BlobStore, ChannelsStorage, DATASETS_SCHEMA, DATASET_ITEMS_SCHEMA, DATASET_VERSIONS_SCHEMA, DatasetsStorage, EXPERIMENTS_SCHEMA, EXPERIMENT_RESULTS_SCHEMA, ExperimentsStorage, FAVORITES_SCHEMA, FavoritesStorage, KNOWLEDGE_ACTIVITY_SCHEMA, KNOWLEDGE_MENTIONS_SCHEMA, KNOWLEDGE_NODES_SCHEMA, KNOWLEDGE_RECORDS_SCHEMA, KNOWLEDGE_SEMANTIC_OUTBOX_SCHEMA, KnowledgeConflictError, KnowledgeNotFoundError, KnowledgeStorage, MCPClientsStorage, MCPServersStorage, MCP_CLIENTS_SCHEMA, MCP_CLIENT_VERSIONS_SCHEMA, MCP_SERVERS_SCHEMA, MCP_SERVER_VERSIONS_SCHEMA, MastraCompositeStore, MemoryStorage, OBSERVATIONAL_MEMORY_TABLE_SCHEMA, ObservabilityStorage, PROMPT_BLOCKS_SCHEMA, PROMPT_BLOCK_VERSIONS_SCHEMA, PromptBlocksStorage, SCORERS_SCHEMA, SCORER_DEFINITIONS_SCHEMA, SCORER_DEFINITION_VERSIONS_SCHEMA, SKILLS_SCHEMA, SKILL_BLOBS_SCHEMA, SKILL_VERSIONS_SCHEMA, SPAN_SCHEMA, SchedulesStorage, ScorerDefinitionsStorage, ScoresStorage, SkillsStorage, StoreOperations, TABLE_AGENTS, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_CHANNEL_CONFIG, TABLE_CHANNEL_INSTALLATIONS, TABLE_CONFIGS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_KNOWLEDGE_ACTIVITY, TABLE_KNOWLEDGE_MENTIONS, TABLE_KNOWLEDGE_NODES, TABLE_KNOWLEDGE_RECORDS, TABLE_KNOWLEDGE_SEMANTIC_OUTBOX, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TOOL_PROVIDER_CONNECTIONS_SCHEMA, ToolProviderConnectionsStorage, TraceStatus, WORKFLOW_DEFINITIONS_SCHEMA, WORKSPACES_SCHEMA, WORKSPACE_VERSIONS_SCHEMA, WorkflowDefinitionsStorage, WorkflowsStorage, WorkspacesStorage, calculatePagination, canonicalizeKnowledgeScope, createKnowledgeUlid, createStorageErrorId, executeRetentionPrune, hasErrorCode, isKnowledgeScopeVisible, knowledgeScopeKey, knowledgeSemanticDocumentId, knowledgeSemanticIdempotencyKey, listTracesArgsSchema, matchesExpectedWorkflowStatus, normalizePerPage, normalizeScheduleTarget, parseKnowledgeNodeCursor, parseKnowledgeWikilinks, resolveListOrderBy, resolveRetentionTargets, retentionCutoffMs, toTraceSpans, validateStorageMetadataFilter } from "@mastra/core/storage";
|
|
3
3
|
import { createPool } from "mysql2/promise";
|
|
4
4
|
import { parseSqlIdentifier } from "@mastra/core/utils";
|
|
5
|
+
import { createHash } from "crypto";
|
|
5
6
|
import { MessageList } from "@mastra/core/agent";
|
|
6
7
|
import { skillSnapshotFieldValuesEqual } from "@mastra/core/storage/domains/skills";
|
|
7
8
|
//#region src/storage/db/schema-snapshot.ts
|
|
@@ -6021,9 +6022,108 @@ var MCPServersMySQL = class MCPServersMySQL extends MCPServersStorage {
|
|
|
6021
6022
|
}
|
|
6022
6023
|
};
|
|
6023
6024
|
//#endregion
|
|
6025
|
+
//#region ../../packages/_internal-core/dist/memory/index.js
|
|
6026
|
+
/**
|
|
6027
|
+
* Shared rules for observational memory lifecycle writes. Every storage adapter applies
|
|
6028
|
+
* these inside its own atomic section (lock, transaction, or conditional update), so the
|
|
6029
|
+
* decisions are identical across adapters.
|
|
6030
|
+
*/
|
|
6031
|
+
/**
|
|
6032
|
+
* Observational memory columns that lifecycle writes depend on. Storage adapters merge these
|
|
6033
|
+
* into the table schema they get from `@mastra/core`, so the columns exist even when the
|
|
6034
|
+
* installed core predates them.
|
|
6035
|
+
*/
|
|
6036
|
+
const OBSERVATIONAL_MEMORY_LIFECYCLE_COLUMNS = { supersededBy: {
|
|
6037
|
+
type: "text",
|
|
6038
|
+
nullable: true
|
|
6039
|
+
} };
|
|
6040
|
+
/** `schema` plus {@link OBSERVATIONAL_MEMORY_LIFECYCLE_COLUMNS}; `undefined` when `schema` is. */
|
|
6041
|
+
function withObservationalMemoryLifecycleColumns(schema) {
|
|
6042
|
+
return schema ? {
|
|
6043
|
+
...schema,
|
|
6044
|
+
...OBSERVATIONAL_MEMORY_LIFECYCLE_COLUMNS
|
|
6045
|
+
} : void 0;
|
|
6046
|
+
}
|
|
6047
|
+
/**
|
|
6048
|
+
* A buffered chunk stores `lastObservedAt = max message time + 1ms`. It is wholly covered
|
|
6049
|
+
* (every message it observed is already at or before the cursor) iff
|
|
6050
|
+
* `cursor >= chunk.lastObservedAt - 1ms`.
|
|
6051
|
+
*/
|
|
6052
|
+
function isBufferedChunkCoveredByCursor(chunkLastObservedAt, cursor) {
|
|
6053
|
+
if (!cursor) return false;
|
|
6054
|
+
return new Date(cursor).getTime() >= new Date(chunkLastObservedAt).getTime() - 1;
|
|
6055
|
+
}
|
|
6056
|
+
/** The later of two cursors; `undefined` only when both are absent. */
|
|
6057
|
+
function maxObservationCursor(a, b) {
|
|
6058
|
+
if (!a) return b ? new Date(b) : void 0;
|
|
6059
|
+
if (!b) return new Date(a);
|
|
6060
|
+
return new Date(Math.max(new Date(a).getTime(), new Date(b).getTime()));
|
|
6061
|
+
}
|
|
6062
|
+
/**
|
|
6063
|
+
* Decide the new generation's text when a reflection built from `snapshot` commits against
|
|
6064
|
+
* the stored record.
|
|
6065
|
+
*
|
|
6066
|
+
* - Stored text equals the snapshot: the reflection replaces it.
|
|
6067
|
+
* - Stored text extends the snapshot (activation only appends): the appended tail is kept
|
|
6068
|
+
* after the reflection, and its tokens are added.
|
|
6069
|
+
* - Anything else (a non-append rewrite): `null` — the reflection must not be applied.
|
|
6070
|
+
*/
|
|
6071
|
+
function planReflectionGenerationText(input) {
|
|
6072
|
+
const stored = input.storedObservations ?? "";
|
|
6073
|
+
const snapshot = input.snapshotObservations ?? "";
|
|
6074
|
+
if (stored === snapshot) return {
|
|
6075
|
+
mode: "equal",
|
|
6076
|
+
observations: input.reflection,
|
|
6077
|
+
tokenCount: input.tokenCount
|
|
6078
|
+
};
|
|
6079
|
+
if (!stored.startsWith(snapshot)) return null;
|
|
6080
|
+
const tail = stored.slice(snapshot.length).trimStart();
|
|
6081
|
+
if (!tail.trim()) return {
|
|
6082
|
+
mode: "equal",
|
|
6083
|
+
observations: input.reflection,
|
|
6084
|
+
tokenCount: input.tokenCount
|
|
6085
|
+
};
|
|
6086
|
+
return {
|
|
6087
|
+
mode: "append",
|
|
6088
|
+
observations: input.reflection ? `${input.reflection}\n\n${tail}` : tail,
|
|
6089
|
+
tokenCount: input.tokenCount + Math.max(0, (input.storedObservationTokenCount ?? 0) - (input.snapshotObservationTokenCount ?? 0))
|
|
6090
|
+
};
|
|
6091
|
+
}
|
|
6092
|
+
/**
|
|
6093
|
+
* Backs `updateActiveObservations` in adapters that implement `commitActiveObservations`.
|
|
6094
|
+
* Callers of `updateActiveObservations` (which resolves `void`) can't see a commit result, so a
|
|
6095
|
+
* write that was not applied throws instead of resolving as if it had committed.
|
|
6096
|
+
*/
|
|
6097
|
+
function assertActiveObservationsApplied(result, id) {
|
|
6098
|
+
if (result.applied) return;
|
|
6099
|
+
throw new Error(result.reason === "retired" ? `Observational memory record ${id} was superseded by a newer generation; observations were not written` : `Observational memory record ${id} changed since the observations were composed; observations were not written`);
|
|
6100
|
+
}
|
|
6101
|
+
/**
|
|
6102
|
+
* Whether the stored text can still accept a reflection built from `snapshot`
|
|
6103
|
+
* (equal, or only appended to since).
|
|
6104
|
+
*/
|
|
6105
|
+
function isAppendOnlySince(storedObservations, snapshotObservations) {
|
|
6106
|
+
const stored = storedObservations ?? "";
|
|
6107
|
+
const snapshot = snapshotObservations ?? "";
|
|
6108
|
+
return stored === snapshot || stored.startsWith(snapshot);
|
|
6109
|
+
}
|
|
6110
|
+
//#endregion
|
|
6024
6111
|
//#region src/storage/domains/memory/index.ts
|
|
6025
6112
|
const OM_TABLE = "mastra_observational_memory";
|
|
6026
6113
|
const OM_TABLE_QUOTED = quoteIdentifier(OM_TABLE, "table name");
|
|
6114
|
+
/** Canonical head order: newest generation, then earliest createdAt, then lowest id. */
|
|
6115
|
+
const OM_HEAD_ORDER = `${quoteIdentifier("generationCount", "column name")} DESC, ${quoteIdentifier("createdAt", "column name")} ASC, ${quoteIdentifier("id", "column name")} ASC`;
|
|
6116
|
+
/** Max `supersededBy` hops when redirecting a write aimed at a retired record. */
|
|
6117
|
+
const OM_MAX_HEAD_HOPS = 3;
|
|
6118
|
+
/** Attempts for an OM transaction that hits a deadlock or lock-wait timeout. */
|
|
6119
|
+
const OM_MAX_TRANSACTION_ATTEMPTS = 3;
|
|
6120
|
+
/** How long initialization waits for the per-key `GET_LOCK` before failing. */
|
|
6121
|
+
const OM_INIT_LOCK_TIMEOUT_SECONDS = 30;
|
|
6122
|
+
/** ER_LOCK_DEADLOCK (1213) or ER_LOCK_WAIT_TIMEOUT (1205): the transaction was rolled back and can be retried. */
|
|
6123
|
+
function isRetryableLockError(error) {
|
|
6124
|
+
const errno = error?.errno;
|
|
6125
|
+
return errno === 1213 || errno === 1205;
|
|
6126
|
+
}
|
|
6027
6127
|
function emitValidationError(message) {
|
|
6028
6128
|
return new MastraError({
|
|
6029
6129
|
id: "MYSQL_MEMORY_VALIDATION_ERROR",
|
|
@@ -6130,7 +6230,7 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
6130
6230
|
tableName: TABLE_RESOURCES,
|
|
6131
6231
|
schema: TABLE_SCHEMAS[TABLE_RESOURCES]
|
|
6132
6232
|
});
|
|
6133
|
-
const omSchema = OBSERVATIONAL_MEMORY_TABLE_SCHEMA?.[OM_TABLE];
|
|
6233
|
+
const omSchema = withObservationalMemoryLifecycleColumns(OBSERVATIONAL_MEMORY_TABLE_SCHEMA?.[OM_TABLE]);
|
|
6134
6234
|
if (omSchema) {
|
|
6135
6235
|
await this.operations.createTable({
|
|
6136
6236
|
tableName: OM_TABLE,
|
|
@@ -6153,7 +6253,8 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
6153
6253
|
"isBufferingObservation",
|
|
6154
6254
|
"isBufferingReflection",
|
|
6155
6255
|
"lastBufferedAtTokens",
|
|
6156
|
-
"lastBufferedAtTime"
|
|
6256
|
+
"lastBufferedAtTime",
|
|
6257
|
+
"supersededBy"
|
|
6157
6258
|
]
|
|
6158
6259
|
});
|
|
6159
6260
|
}
|
|
@@ -6171,6 +6272,7 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
6171
6272
|
if (err?.errno !== 1061) throw err;
|
|
6172
6273
|
}
|
|
6173
6274
|
}
|
|
6275
|
+
if (omSchema) await this.backfillSupersededBy();
|
|
6174
6276
|
await this.createDefaultIndexes();
|
|
6175
6277
|
await this.createCustomIndexes();
|
|
6176
6278
|
}
|
|
@@ -7287,7 +7389,7 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7287
7389
|
async getObservationalMemory(threadId, resourceId) {
|
|
7288
7390
|
try {
|
|
7289
7391
|
const lookupKey = this.getOMKey(threadId, resourceId);
|
|
7290
|
-
const [rows] = await this.pool.execute(`SELECT * FROM ${OM_TABLE_QUOTED} WHERE ${omCol("lookupKey")} = ? ORDER BY ${
|
|
7392
|
+
const [rows] = await this.pool.execute(`SELECT * FROM ${OM_TABLE_QUOTED} WHERE ${omCol("lookupKey")} = ? ORDER BY ${OM_HEAD_ORDER} LIMIT 1`, [lookupKey]);
|
|
7291
7393
|
if (!rows || rows.length === 0) return null;
|
|
7292
7394
|
return this.parseOMRow(rows[0]);
|
|
7293
7395
|
} catch (error) {
|
|
@@ -7349,32 +7451,11 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7349
7451
|
}
|
|
7350
7452
|
async initializeObservationalMemory(input) {
|
|
7351
7453
|
try {
|
|
7454
|
+
const lookupKey = this.getOMKey(input.threadId, input.resourceId);
|
|
7352
7455
|
const id = globalThis.crypto.randomUUID();
|
|
7353
7456
|
const now = /* @__PURE__ */ new Date();
|
|
7354
|
-
const
|
|
7355
|
-
|
|
7356
|
-
id,
|
|
7357
|
-
scope: input.scope,
|
|
7358
|
-
threadId: input.threadId,
|
|
7359
|
-
resourceId: input.resourceId,
|
|
7360
|
-
createdAt: now,
|
|
7361
|
-
updatedAt: now,
|
|
7362
|
-
lastObservedAt: void 0,
|
|
7363
|
-
originType: "initial",
|
|
7364
|
-
generationCount: 0,
|
|
7365
|
-
activeObservations: "",
|
|
7366
|
-
totalTokensObserved: 0,
|
|
7367
|
-
observationTokenCount: 0,
|
|
7368
|
-
pendingMessageTokens: 0,
|
|
7369
|
-
isReflecting: false,
|
|
7370
|
-
isObserving: false,
|
|
7371
|
-
isBufferingObservation: false,
|
|
7372
|
-
isBufferingReflection: false,
|
|
7373
|
-
lastBufferedAtTokens: 0,
|
|
7374
|
-
lastBufferedAtTime: null,
|
|
7375
|
-
config: input.config,
|
|
7376
|
-
observedTimezone: input.observedTimezone
|
|
7377
|
-
};
|
|
7457
|
+
const existing = await this.getObservationalMemory(input.threadId, input.resourceId);
|
|
7458
|
+
if (existing) return existing;
|
|
7378
7459
|
const nowSql = transformToSqlValue(now);
|
|
7379
7460
|
const cols = [
|
|
7380
7461
|
"id",
|
|
@@ -7400,36 +7481,53 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7400
7481
|
"lastBufferedAtTime",
|
|
7401
7482
|
"observedTimezone",
|
|
7402
7483
|
"createdAt",
|
|
7403
|
-
"updatedAt"
|
|
7484
|
+
"updatedAt",
|
|
7485
|
+
"supersededBy"
|
|
7404
7486
|
].map(omCol).join(", ");
|
|
7405
|
-
const placeholders = Array.from({ length:
|
|
7406
|
-
|
|
7407
|
-
|
|
7408
|
-
|
|
7409
|
-
|
|
7410
|
-
|
|
7411
|
-
|
|
7412
|
-
|
|
7413
|
-
|
|
7414
|
-
|
|
7415
|
-
|
|
7416
|
-
|
|
7417
|
-
|
|
7418
|
-
|
|
7419
|
-
|
|
7420
|
-
|
|
7421
|
-
|
|
7422
|
-
|
|
7423
|
-
|
|
7424
|
-
|
|
7425
|
-
|
|
7426
|
-
|
|
7427
|
-
|
|
7428
|
-
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7432
|
-
|
|
7487
|
+
const placeholders = Array.from({ length: 25 }, () => "?").join(", ");
|
|
7488
|
+
const lockName = `mastra_om_init:${createHash("sha256").update(lookupKey).digest("hex").slice(0, 40)}`;
|
|
7489
|
+
const connection = await this.pool.getConnection();
|
|
7490
|
+
try {
|
|
7491
|
+
const [lockRows] = await connection.query("SELECT GET_LOCK(?, ?) AS acquired", [lockName, OM_INIT_LOCK_TIMEOUT_SECONDS]);
|
|
7492
|
+
if (Number(lockRows[0]?.acquired) !== 1) throw new Error(`Timed out waiting for the observational memory initialization lock for ${lookupKey}`);
|
|
7493
|
+
try {
|
|
7494
|
+
const [liveRows] = await connection.query(`SELECT 1 FROM ${OM_TABLE_QUOTED} WHERE ${omCol("lookupKey")} = ? AND ${omCol("supersededBy")} IS NULL LIMIT 1`, [lookupKey]);
|
|
7495
|
+
if (liveRows.length === 0) await connection.execute(`INSERT INTO ${OM_TABLE_QUOTED} (${cols}) VALUES (${placeholders})`, [
|
|
7496
|
+
id,
|
|
7497
|
+
lookupKey,
|
|
7498
|
+
input.scope,
|
|
7499
|
+
input.resourceId,
|
|
7500
|
+
input.threadId || null,
|
|
7501
|
+
"",
|
|
7502
|
+
null,
|
|
7503
|
+
"initial",
|
|
7504
|
+
JSON.stringify(input.config),
|
|
7505
|
+
0,
|
|
7506
|
+
null,
|
|
7507
|
+
null,
|
|
7508
|
+
0,
|
|
7509
|
+
0,
|
|
7510
|
+
0,
|
|
7511
|
+
false,
|
|
7512
|
+
false,
|
|
7513
|
+
false,
|
|
7514
|
+
false,
|
|
7515
|
+
0,
|
|
7516
|
+
null,
|
|
7517
|
+
input.observedTimezone || null,
|
|
7518
|
+
nowSql,
|
|
7519
|
+
nowSql,
|
|
7520
|
+
null
|
|
7521
|
+
]);
|
|
7522
|
+
} finally {
|
|
7523
|
+
await connection.query("SELECT RELEASE_LOCK(?)", [lockName]);
|
|
7524
|
+
}
|
|
7525
|
+
} finally {
|
|
7526
|
+
connection.release();
|
|
7527
|
+
}
|
|
7528
|
+
const head = await this.getObservationalMemory(input.threadId, input.resourceId);
|
|
7529
|
+
if (!head) throwOMNotFound(id, "INITIALIZE_OBSERVATIONAL_MEMORY");
|
|
7530
|
+
return head;
|
|
7433
7531
|
} catch (error) {
|
|
7434
7532
|
rethrowOrWrapOM(error, "", "INITIALIZE_OBSERVATIONAL_MEMORY", {
|
|
7435
7533
|
threadId: input.threadId,
|
|
@@ -7438,115 +7536,65 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7438
7536
|
}
|
|
7439
7537
|
}
|
|
7440
7538
|
async updateActiveObservations(input) {
|
|
7539
|
+
assertActiveObservationsApplied(await this.commitActiveObservations(input), input.id);
|
|
7540
|
+
}
|
|
7541
|
+
async commitActiveObservations(input) {
|
|
7441
7542
|
try {
|
|
7442
|
-
const now = /* @__PURE__ */ new Date();
|
|
7443
7543
|
const observedMessageIdsJson = input.observedMessageIds ? JSON.stringify(input.observedMessageIds) : null;
|
|
7444
|
-
|
|
7445
|
-
|
|
7446
|
-
|
|
7447
|
-
|
|
7448
|
-
|
|
7449
|
-
|
|
7450
|
-
|
|
7451
|
-
|
|
7452
|
-
|
|
7453
|
-
|
|
7454
|
-
|
|
7455
|
-
input.
|
|
7456
|
-
|
|
7457
|
-
|
|
7458
|
-
|
|
7459
|
-
|
|
7460
|
-
|
|
7461
|
-
|
|
7544
|
+
return await this.withOMTransaction(async (connection) => {
|
|
7545
|
+
const row = await this.lockOMRow(connection, input.id);
|
|
7546
|
+
if (!row) throwOMNotFound(input.id, "UPDATE_ACTIVE_OBSERVATIONS");
|
|
7547
|
+
if (row.supersededBy) return {
|
|
7548
|
+
applied: false,
|
|
7549
|
+
reason: "retired"
|
|
7550
|
+
};
|
|
7551
|
+
if (input.expectedActiveObservations !== void 0 && input.expectedActiveObservations !== (row.activeObservations || "")) return {
|
|
7552
|
+
applied: false,
|
|
7553
|
+
reason: "conflict"
|
|
7554
|
+
};
|
|
7555
|
+
const lastObservedAt = maxObservationCursor(parseDateTime(row.lastObservedAt), input.lastObservedAt);
|
|
7556
|
+
await connection.execute(`UPDATE ${OM_TABLE_QUOTED} SET
|
|
7557
|
+
${omCol("activeObservations")} = ?,
|
|
7558
|
+
${omCol("lastObservedAt")} = ?,
|
|
7559
|
+
${omCol("pendingMessageTokens")} = 0,
|
|
7560
|
+
${omCol("observationTokenCount")} = ?,
|
|
7561
|
+
${omCol("totalTokensObserved")} = ${omCol("totalTokensObserved")} + ?,
|
|
7562
|
+
${omCol("observedMessageIds")} = ?,
|
|
7563
|
+
${omCol("updatedAt")} = ?
|
|
7564
|
+
WHERE ${omCol("id")} = ?`, [
|
|
7565
|
+
input.observations,
|
|
7566
|
+
transformToSqlValue(lastObservedAt),
|
|
7567
|
+
input.tokenCount,
|
|
7568
|
+
input.tokenCount,
|
|
7569
|
+
observedMessageIdsJson,
|
|
7570
|
+
transformToSqlValue(/* @__PURE__ */ new Date()),
|
|
7571
|
+
input.id
|
|
7572
|
+
]);
|
|
7573
|
+
return { applied: true };
|
|
7574
|
+
});
|
|
7462
7575
|
} catch (error) {
|
|
7463
7576
|
rethrowOrWrapOM(error, input.id, "UPDATE_ACTIVE_OBSERVATIONS");
|
|
7464
7577
|
}
|
|
7465
7578
|
}
|
|
7466
7579
|
async createReflectionGeneration(input) {
|
|
7467
7580
|
try {
|
|
7468
|
-
const
|
|
7469
|
-
|
|
7470
|
-
|
|
7471
|
-
|
|
7472
|
-
|
|
7473
|
-
|
|
7474
|
-
|
|
7475
|
-
|
|
7476
|
-
|
|
7477
|
-
|
|
7478
|
-
|
|
7479
|
-
|
|
7480
|
-
|
|
7481
|
-
|
|
7482
|
-
|
|
7483
|
-
|
|
7484
|
-
|
|
7485
|
-
isReflecting: false,
|
|
7486
|
-
isObserving: false,
|
|
7487
|
-
isBufferingObservation: false,
|
|
7488
|
-
isBufferingReflection: false,
|
|
7489
|
-
lastBufferedAtTokens: 0,
|
|
7490
|
-
lastBufferedAtTime: null,
|
|
7491
|
-
config: input.currentRecord.config,
|
|
7492
|
-
metadata: input.currentRecord.metadata,
|
|
7493
|
-
observedTimezone: input.currentRecord.observedTimezone
|
|
7494
|
-
};
|
|
7495
|
-
const nowSql = transformToSqlValue(now);
|
|
7496
|
-
const cols = [
|
|
7497
|
-
"id",
|
|
7498
|
-
"lookupKey",
|
|
7499
|
-
"scope",
|
|
7500
|
-
"resourceId",
|
|
7501
|
-
"threadId",
|
|
7502
|
-
"activeObservations",
|
|
7503
|
-
"activeObservationsPendingUpdate",
|
|
7504
|
-
"originType",
|
|
7505
|
-
"config",
|
|
7506
|
-
"generationCount",
|
|
7507
|
-
"lastObservedAt",
|
|
7508
|
-
"lastReflectionAt",
|
|
7509
|
-
"pendingMessageTokens",
|
|
7510
|
-
"totalTokensObserved",
|
|
7511
|
-
"observationTokenCount",
|
|
7512
|
-
"isObserving",
|
|
7513
|
-
"isReflecting",
|
|
7514
|
-
"isBufferingObservation",
|
|
7515
|
-
"isBufferingReflection",
|
|
7516
|
-
"lastBufferedAtTokens",
|
|
7517
|
-
"lastBufferedAtTime",
|
|
7518
|
-
"observedTimezone",
|
|
7519
|
-
"createdAt",
|
|
7520
|
-
"updatedAt"
|
|
7521
|
-
].map(omCol).join(", ");
|
|
7522
|
-
const placeholders = Array.from({ length: 24 }, () => "?").join(", ");
|
|
7523
|
-
await this.pool.execute(`INSERT INTO ${OM_TABLE_QUOTED} (${cols}) VALUES (${placeholders})`, [
|
|
7524
|
-
id,
|
|
7525
|
-
lookupKey,
|
|
7526
|
-
record.scope,
|
|
7527
|
-
record.resourceId,
|
|
7528
|
-
record.threadId || null,
|
|
7529
|
-
input.reflection,
|
|
7530
|
-
null,
|
|
7531
|
-
"reflection",
|
|
7532
|
-
JSON.stringify(record.config),
|
|
7533
|
-
input.currentRecord.generationCount + 1,
|
|
7534
|
-
record.lastObservedAt ? transformToSqlValue(record.lastObservedAt) : null,
|
|
7535
|
-
nowSql,
|
|
7536
|
-
record.pendingMessageTokens,
|
|
7537
|
-
record.totalTokensObserved,
|
|
7538
|
-
record.observationTokenCount,
|
|
7539
|
-
false,
|
|
7540
|
-
false,
|
|
7541
|
-
false,
|
|
7542
|
-
false,
|
|
7543
|
-
0,
|
|
7544
|
-
null,
|
|
7545
|
-
record.observedTimezone || null,
|
|
7546
|
-
nowSql,
|
|
7547
|
-
nowSql
|
|
7548
|
-
]);
|
|
7549
|
-
return record;
|
|
7581
|
+
const { currentRecord } = input;
|
|
7582
|
+
return await this.withOMTransaction(async (connection) => {
|
|
7583
|
+
const row = await this.lockOMRow(connection, currentRecord.id);
|
|
7584
|
+
if (!row) return currentRecord;
|
|
7585
|
+
if (row.supersededBy) return this.parseOMRow(await this.resolveLiveOMRow(connection, row));
|
|
7586
|
+
const stored = this.parseOMRow(row);
|
|
7587
|
+
const plan = planReflectionGenerationText({
|
|
7588
|
+
storedObservations: stored.activeObservations,
|
|
7589
|
+
storedObservationTokenCount: stored.observationTokenCount,
|
|
7590
|
+
snapshotObservations: currentRecord.activeObservations,
|
|
7591
|
+
snapshotObservationTokenCount: currentRecord.observationTokenCount,
|
|
7592
|
+
reflection: input.reflection,
|
|
7593
|
+
tokenCount: input.tokenCount
|
|
7594
|
+
});
|
|
7595
|
+
if (!plan) return stored;
|
|
7596
|
+
return this.rollOverOMRow(connection, stored.id, plan.observations, plan.tokenCount, input.newRecordId, false);
|
|
7597
|
+
});
|
|
7550
7598
|
} catch (error) {
|
|
7551
7599
|
rethrowOrWrapOM(error, input.currentRecord.id, "CREATE_REFLECTION_GENERATION", { currentRecordId: input.currentRecord.id });
|
|
7552
7600
|
}
|
|
@@ -7559,20 +7607,22 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7559
7607
|
}
|
|
7560
7608
|
async setBufferingObservationFlag(id, isBuffering, lastBufferedAtTokens) {
|
|
7561
7609
|
try {
|
|
7562
|
-
|
|
7563
|
-
|
|
7564
|
-
|
|
7565
|
-
|
|
7566
|
-
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
7570
|
-
|
|
7571
|
-
|
|
7572
|
-
|
|
7573
|
-
|
|
7574
|
-
|
|
7575
|
-
|
|
7610
|
+
await this.withOMTransaction(async (connection) => {
|
|
7611
|
+
const row = await this.lockOMRow(connection, id);
|
|
7612
|
+
if (!row) throwOMNotFound(id, "SET_BUFFERING_OBSERVATION_FLAG");
|
|
7613
|
+
const target = await this.resolveLiveOMRow(connection, row);
|
|
7614
|
+
const sets = [
|
|
7615
|
+
`${omCol("isBufferingObservation")} = ?`,
|
|
7616
|
+
...lastBufferedAtTokens !== void 0 ? [`${omCol("lastBufferedAtTokens")} = ?`] : [],
|
|
7617
|
+
`${omCol("updatedAt")} = ?`
|
|
7618
|
+
].join(", ");
|
|
7619
|
+
await connection.execute(`UPDATE ${OM_TABLE_QUOTED} SET ${sets} WHERE ${omCol("id")} = ?`, [
|
|
7620
|
+
isBuffering,
|
|
7621
|
+
...lastBufferedAtTokens !== void 0 ? [lastBufferedAtTokens] : [],
|
|
7622
|
+
transformToSqlValue(/* @__PURE__ */ new Date()),
|
|
7623
|
+
target.id
|
|
7624
|
+
]);
|
|
7625
|
+
});
|
|
7576
7626
|
} catch (error) {
|
|
7577
7627
|
rethrowOrWrapOM(error, id, "SET_BUFFERING_OBSERVATION_FLAG");
|
|
7578
7628
|
}
|
|
@@ -7593,83 +7643,78 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7593
7643
|
}
|
|
7594
7644
|
async setPendingMessageTokens(id, tokenCount) {
|
|
7595
7645
|
try {
|
|
7596
|
-
|
|
7597
|
-
|
|
7598
|
-
|
|
7599
|
-
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
|
|
7646
|
+
await this.withOMTransaction(async (connection) => {
|
|
7647
|
+
const row = await this.lockOMRow(connection, id);
|
|
7648
|
+
if (!row) throwOMNotFound(id, "SET_PENDING_MESSAGE_TOKENS");
|
|
7649
|
+
const target = await this.resolveLiveOMRow(connection, row);
|
|
7650
|
+
await connection.execute(`UPDATE ${OM_TABLE_QUOTED} SET
|
|
7651
|
+
${omCol("pendingMessageTokens")} = ?,
|
|
7652
|
+
${omCol("updatedAt")} = ?
|
|
7653
|
+
WHERE ${omCol("id")} = ?`, [
|
|
7654
|
+
tokenCount,
|
|
7655
|
+
transformToSqlValue(/* @__PURE__ */ new Date()),
|
|
7656
|
+
target.id
|
|
7657
|
+
]);
|
|
7658
|
+
});
|
|
7605
7659
|
} catch (error) {
|
|
7606
7660
|
rethrowOrWrapOM(error, id, "SET_PENDING_MESSAGE_TOKENS");
|
|
7607
7661
|
}
|
|
7608
7662
|
}
|
|
7609
7663
|
async updateBufferedObservations(input) {
|
|
7610
|
-
|
|
7611
|
-
|
|
7612
|
-
|
|
7613
|
-
|
|
7614
|
-
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7620
|
-
|
|
7621
|
-
|
|
7622
|
-
|
|
7623
|
-
|
|
7624
|
-
|
|
7625
|
-
|
|
7626
|
-
|
|
7627
|
-
|
|
7628
|
-
|
|
7629
|
-
|
|
7630
|
-
|
|
7631
|
-
|
|
7632
|
-
|
|
7633
|
-
|
|
7634
|
-
|
|
7635
|
-
|
|
7664
|
+
await this.appendBufferedObservations(input);
|
|
7665
|
+
}
|
|
7666
|
+
async appendBufferedObservations(input) {
|
|
7667
|
+
try {
|
|
7668
|
+
const newChunk = {
|
|
7669
|
+
id: `ombuf-${globalThis.crypto.randomUUID()}`,
|
|
7670
|
+
cycleId: input.chunk.cycleId,
|
|
7671
|
+
observations: input.chunk.observations,
|
|
7672
|
+
tokenCount: input.chunk.tokenCount,
|
|
7673
|
+
messageIds: input.chunk.messageIds,
|
|
7674
|
+
messageTokens: input.chunk.messageTokens,
|
|
7675
|
+
lastObservedAt: input.chunk.lastObservedAt,
|
|
7676
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
7677
|
+
suggestedContinuation: input.chunk.suggestedContinuation,
|
|
7678
|
+
currentTask: input.chunk.currentTask,
|
|
7679
|
+
threadTitle: input.chunk.threadTitle,
|
|
7680
|
+
extractedValues: input.chunk.extractedValues,
|
|
7681
|
+
extractionFailures: input.chunk.extractionFailures
|
|
7682
|
+
};
|
|
7683
|
+
return await this.withOMTransaction(async (connection) => {
|
|
7684
|
+
const row = await this.lockOMRow(connection, input.id);
|
|
7685
|
+
if (!row) throwOMNotFound(input.id, "UPDATE_BUFFERED_OBSERVATIONS");
|
|
7686
|
+
const target = await this.resolveLiveOMRow(connection, row);
|
|
7687
|
+
const existingChunks = parseBufferedChunks(target.bufferedObservationChunks);
|
|
7688
|
+
if (existingChunks.some((existing) => existing.cycleId === input.chunk.cycleId) || isBufferedChunkCoveredByCursor(input.chunk.lastObservedAt, parseDateTime(target.lastObservedAt))) return {
|
|
7689
|
+
persisted: false,
|
|
7690
|
+
recordId: target.id
|
|
7636
7691
|
};
|
|
7637
|
-
const
|
|
7638
|
-
|
|
7639
|
-
const [result] = await connection.execute(`UPDATE ${OM_TABLE_QUOTED} SET
|
|
7692
|
+
const lastBufferedAtTime = maxObservationCursor(parseDateTime(target.lastBufferedAtTime), input.lastBufferedAtTime ?? void 0);
|
|
7693
|
+
await connection.execute(`UPDATE ${OM_TABLE_QUOTED} SET
|
|
7640
7694
|
${omCol("bufferedObservationChunks")} = ?,
|
|
7641
|
-
${omCol("lastBufferedAtTime")} =
|
|
7695
|
+
${omCol("lastBufferedAtTime")} = ?,
|
|
7642
7696
|
${omCol("updatedAt")} = ?
|
|
7643
7697
|
WHERE ${omCol("id")} = ?`, [
|
|
7644
|
-
JSON.stringify(
|
|
7645
|
-
lastBufferedAtTime,
|
|
7646
|
-
|
|
7647
|
-
|
|
7698
|
+
JSON.stringify([...existingChunks, newChunk]),
|
|
7699
|
+
transformToSqlValue(lastBufferedAtTime),
|
|
7700
|
+
transformToSqlValue(/* @__PURE__ */ new Date()),
|
|
7701
|
+
target.id
|
|
7648
7702
|
]);
|
|
7649
|
-
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7653
|
-
|
|
7654
|
-
} finally {
|
|
7655
|
-
connection.release();
|
|
7656
|
-
}
|
|
7703
|
+
return {
|
|
7704
|
+
persisted: true,
|
|
7705
|
+
recordId: target.id
|
|
7706
|
+
};
|
|
7707
|
+
});
|
|
7657
7708
|
} catch (error) {
|
|
7658
7709
|
rethrowOrWrapOM(error, input.id, "UPDATE_BUFFERED_OBSERVATIONS");
|
|
7659
7710
|
}
|
|
7660
7711
|
}
|
|
7661
7712
|
async swapBufferedToActive(input) {
|
|
7662
|
-
const connection = await this.pool.getConnection();
|
|
7663
7713
|
try {
|
|
7664
|
-
await
|
|
7665
|
-
|
|
7666
|
-
|
|
7667
|
-
|
|
7668
|
-
const row = currentRows[0];
|
|
7669
|
-
const chunks = parseBufferedChunks(row.bufferedObservationChunks);
|
|
7670
|
-
if (chunks.length === 0) {
|
|
7671
|
-
await connection.commit();
|
|
7672
|
-
return {
|
|
7714
|
+
return await this.withOMTransaction(async (connection) => {
|
|
7715
|
+
const row = await this.lockOMRow(connection, input.id);
|
|
7716
|
+
if (!row) throwOMNotFound(input.id, "SWAP_BUFFERED_TO_ACTIVE");
|
|
7717
|
+
const emptyResult = {
|
|
7673
7718
|
chunksActivated: 0,
|
|
7674
7719
|
messageTokensActivated: 0,
|
|
7675
7720
|
observationTokensActivated: 0,
|
|
@@ -7677,95 +7722,103 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7677
7722
|
activatedCycleIds: [],
|
|
7678
7723
|
activatedMessageIds: []
|
|
7679
7724
|
};
|
|
7680
|
-
|
|
7681
|
-
|
|
7682
|
-
|
|
7683
|
-
|
|
7684
|
-
|
|
7685
|
-
|
|
7686
|
-
|
|
7687
|
-
|
|
7688
|
-
|
|
7689
|
-
|
|
7690
|
-
|
|
7691
|
-
|
|
7692
|
-
|
|
7693
|
-
|
|
7694
|
-
|
|
7725
|
+
if (row.supersededBy) return {
|
|
7726
|
+
...emptyResult,
|
|
7727
|
+
retired: true
|
|
7728
|
+
};
|
|
7729
|
+
const persistedChunks = parseBufferedChunks(row.bufferedObservationChunks);
|
|
7730
|
+
const refreshedWeights = new Map((Array.isArray(input.bufferedChunks) ? input.bufferedChunks : []).map((c) => [c.id, c.messageTokens]));
|
|
7731
|
+
const chunks = persistedChunks.map((c) => refreshedWeights.has(c.id) ? {
|
|
7732
|
+
...c,
|
|
7733
|
+
messageTokens: refreshedWeights.get(c.id)
|
|
7734
|
+
} : c);
|
|
7735
|
+
if (chunks.length === 0) return emptyResult;
|
|
7736
|
+
const retentionFloor = input.messageTokensThreshold * (1 - input.activationRatio);
|
|
7737
|
+
const targetMessageTokens = Math.max(0, input.currentPendingTokens - retentionFloor);
|
|
7738
|
+
let cumulativeMessageTokens = 0;
|
|
7739
|
+
let bestOverBoundary = 0;
|
|
7740
|
+
let bestOverTokens = 0;
|
|
7741
|
+
let bestUnderBoundary = 0;
|
|
7742
|
+
let bestUnderTokens = 0;
|
|
7743
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
7744
|
+
cumulativeMessageTokens += chunks[i].messageTokens ?? 0;
|
|
7745
|
+
const boundary = i + 1;
|
|
7746
|
+
if (cumulativeMessageTokens >= targetMessageTokens) {
|
|
7747
|
+
if (bestOverBoundary === 0 || cumulativeMessageTokens < bestOverTokens) {
|
|
7748
|
+
bestOverBoundary = boundary;
|
|
7749
|
+
bestOverTokens = cumulativeMessageTokens;
|
|
7750
|
+
}
|
|
7751
|
+
} else if (cumulativeMessageTokens > bestUnderTokens) {
|
|
7752
|
+
bestUnderBoundary = boundary;
|
|
7753
|
+
bestUnderTokens = cumulativeMessageTokens;
|
|
7695
7754
|
}
|
|
7696
|
-
}
|
|
7697
|
-
|
|
7698
|
-
|
|
7699
|
-
|
|
7700
|
-
|
|
7701
|
-
|
|
7702
|
-
|
|
7703
|
-
|
|
7704
|
-
|
|
7705
|
-
|
|
7706
|
-
|
|
7707
|
-
|
|
7708
|
-
|
|
7709
|
-
|
|
7710
|
-
|
|
7711
|
-
|
|
7712
|
-
|
|
7713
|
-
|
|
7714
|
-
|
|
7715
|
-
|
|
7716
|
-
|
|
7717
|
-
|
|
7718
|
-
|
|
7719
|
-
|
|
7720
|
-
|
|
7721
|
-
|
|
7722
|
-
|
|
7723
|
-
|
|
7724
|
-
|
|
7725
|
-
|
|
7726
|
-
|
|
7727
|
-
|
|
7728
|
-
|
|
7729
|
-
|
|
7730
|
-
|
|
7731
|
-
|
|
7732
|
-
${omCol("
|
|
7733
|
-
|
|
7734
|
-
|
|
7735
|
-
|
|
7736
|
-
|
|
7737
|
-
|
|
7738
|
-
|
|
7739
|
-
|
|
7740
|
-
|
|
7741
|
-
|
|
7742
|
-
|
|
7743
|
-
|
|
7744
|
-
|
|
7745
|
-
|
|
7746
|
-
|
|
7747
|
-
|
|
7748
|
-
|
|
7749
|
-
|
|
7750
|
-
|
|
7751
|
-
|
|
7752
|
-
|
|
7753
|
-
|
|
7754
|
-
|
|
7755
|
-
|
|
7756
|
-
|
|
7757
|
-
|
|
7758
|
-
|
|
7759
|
-
|
|
7760
|
-
|
|
7761
|
-
suggestedContinuation: latestChunkHints?.suggestedContinuation ?? void 0,
|
|
7762
|
-
currentTask: latestChunkHints?.currentTask ?? void 0
|
|
7763
|
-
};
|
|
7755
|
+
}
|
|
7756
|
+
const maxOvershoot = retentionFloor * .95;
|
|
7757
|
+
const overshoot = bestOverTokens - targetMessageTokens;
|
|
7758
|
+
const remainingAfterOver = input.currentPendingTokens - bestOverTokens;
|
|
7759
|
+
const remainingAfterUnder = input.currentPendingTokens - bestUnderTokens;
|
|
7760
|
+
const minRemaining = Math.min(1e3, retentionFloor);
|
|
7761
|
+
let chunksToActivate;
|
|
7762
|
+
if (input.forceMaxActivation && bestOverBoundary > 0 && remainingAfterOver >= minRemaining) chunksToActivate = bestOverBoundary;
|
|
7763
|
+
else if (bestOverBoundary > 0 && overshoot <= maxOvershoot && remainingAfterOver >= minRemaining) chunksToActivate = bestOverBoundary;
|
|
7764
|
+
else if (bestUnderBoundary > 0 && remainingAfterUnder >= minRemaining) chunksToActivate = bestUnderBoundary;
|
|
7765
|
+
else if (bestOverBoundary > 0) chunksToActivate = bestOverBoundary;
|
|
7766
|
+
else chunksToActivate = 1;
|
|
7767
|
+
const activatedChunks = chunks.slice(0, chunksToActivate);
|
|
7768
|
+
const remainingChunks = persistedChunks.slice(chunksToActivate);
|
|
7769
|
+
const activatedContent = activatedChunks.map((c) => c.observations).join("\n\n");
|
|
7770
|
+
const activatedTokens = activatedChunks.reduce((sum, c) => sum + c.tokenCount, 0);
|
|
7771
|
+
const activatedMessageTokens = activatedChunks.reduce((sum, c) => sum + (c.messageTokens ?? 0), 0);
|
|
7772
|
+
const activatedMessageCount = activatedChunks.reduce((sum, c) => sum + c.messageIds.length, 0);
|
|
7773
|
+
const activatedCycleIds = activatedChunks.map((c) => c.cycleId).filter((cid) => !!cid);
|
|
7774
|
+
const activatedMessageIds = activatedChunks.flatMap((c) => c.messageIds ?? []);
|
|
7775
|
+
const latestChunk = activatedChunks[activatedChunks.length - 1];
|
|
7776
|
+
const derivedLastObservedAt = input.lastObservedAt ?? (latestChunk?.lastObservedAt ? new Date(latestChunk.lastObservedAt) : /* @__PURE__ */ new Date());
|
|
7777
|
+
const lastObservedAt = maxObservationCursor(parseDateTime(row.lastObservedAt), derivedLastObservedAt);
|
|
7778
|
+
const existingActive = row.activeObservations || "";
|
|
7779
|
+
const existingTokenCount = Number(row.observationTokenCount || 0);
|
|
7780
|
+
const newActive = existingActive ? `${existingActive}\n\n${activatedContent}` : activatedContent;
|
|
7781
|
+
const newTokenCount = existingTokenCount + activatedTokens;
|
|
7782
|
+
const existingPending = Number(row.pendingMessageTokens || 0);
|
|
7783
|
+
const newPending = Math.max(0, existingPending - activatedMessageTokens);
|
|
7784
|
+
await connection.execute(`UPDATE ${OM_TABLE_QUOTED} SET
|
|
7785
|
+
${omCol("activeObservations")} = ?,
|
|
7786
|
+
${omCol("observationTokenCount")} = ?,
|
|
7787
|
+
${omCol("pendingMessageTokens")} = ?,
|
|
7788
|
+
${omCol("bufferedObservationChunks")} = ?,
|
|
7789
|
+
${omCol("lastObservedAt")} = ?,
|
|
7790
|
+
${omCol("updatedAt")} = ?
|
|
7791
|
+
WHERE ${omCol("id")} = ?`, [
|
|
7792
|
+
newActive,
|
|
7793
|
+
newTokenCount,
|
|
7794
|
+
newPending,
|
|
7795
|
+
remainingChunks.length > 0 ? JSON.stringify(remainingChunks) : null,
|
|
7796
|
+
transformToSqlValue(lastObservedAt),
|
|
7797
|
+
transformToSqlValue(/* @__PURE__ */ new Date()),
|
|
7798
|
+
input.id
|
|
7799
|
+
]);
|
|
7800
|
+
const latestChunkHints = activatedChunks[activatedChunks.length - 1];
|
|
7801
|
+
return {
|
|
7802
|
+
chunksActivated: activatedChunks.length,
|
|
7803
|
+
messageTokensActivated: activatedMessageTokens,
|
|
7804
|
+
observationTokensActivated: activatedTokens,
|
|
7805
|
+
messagesActivated: activatedMessageCount,
|
|
7806
|
+
activatedCycleIds,
|
|
7807
|
+
activatedMessageIds,
|
|
7808
|
+
observations: activatedContent,
|
|
7809
|
+
perChunk: activatedChunks.map((c) => ({
|
|
7810
|
+
cycleId: c.cycleId ?? "",
|
|
7811
|
+
messageTokens: c.messageTokens ?? 0,
|
|
7812
|
+
observationTokens: c.tokenCount,
|
|
7813
|
+
messageCount: c.messageIds.length,
|
|
7814
|
+
observations: c.observations
|
|
7815
|
+
})),
|
|
7816
|
+
suggestedContinuation: latestChunkHints?.suggestedContinuation ?? void 0,
|
|
7817
|
+
currentTask: latestChunkHints?.currentTask ?? void 0
|
|
7818
|
+
};
|
|
7819
|
+
});
|
|
7764
7820
|
} catch (error) {
|
|
7765
|
-
await connection.rollback();
|
|
7766
7821
|
rethrowOrWrapOM(error, input.id, "SWAP_BUFFERED_TO_ACTIVE");
|
|
7767
|
-
} finally {
|
|
7768
|
-
connection.release();
|
|
7769
7822
|
}
|
|
7770
7823
|
}
|
|
7771
7824
|
async updateBufferedReflection(input) {
|
|
@@ -7798,38 +7851,182 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7798
7851
|
}
|
|
7799
7852
|
async swapBufferedReflectionToActive(input) {
|
|
7800
7853
|
try {
|
|
7801
|
-
const
|
|
7802
|
-
|
|
7803
|
-
|
|
7804
|
-
|
|
7805
|
-
|
|
7806
|
-
|
|
7807
|
-
|
|
7808
|
-
|
|
7809
|
-
|
|
7810
|
-
|
|
7811
|
-
|
|
7812
|
-
|
|
7813
|
-
|
|
7814
|
-
|
|
7815
|
-
|
|
7816
|
-
|
|
7817
|
-
|
|
7818
|
-
|
|
7854
|
+
const { currentRecord } = input;
|
|
7855
|
+
return await this.withOMTransaction(async (connection) => {
|
|
7856
|
+
const row = await this.lockOMRow(connection, currentRecord.id);
|
|
7857
|
+
if (!row) return currentRecord;
|
|
7858
|
+
if (row.supersededBy) return this.parseOMRow(await this.resolveLiveOMRow(connection, row));
|
|
7859
|
+
const stored = this.parseOMRow(row);
|
|
7860
|
+
const bufferedReflection = stored.bufferedReflection || "";
|
|
7861
|
+
if (!bufferedReflection) throw new MastraError({
|
|
7862
|
+
id: createStorageErrorId("MYSQL", "SWAP_BUFFERED_REFLECTION_TO_ACTIVE", "NO_CONTENT"),
|
|
7863
|
+
text: "No buffered reflection to swap",
|
|
7864
|
+
domain: ErrorDomain.STORAGE,
|
|
7865
|
+
category: ErrorCategory.USER,
|
|
7866
|
+
details: { id: currentRecord.id }
|
|
7867
|
+
});
|
|
7868
|
+
if (!isAppendOnlySince(stored.activeObservations, currentRecord.activeObservations)) return stored;
|
|
7869
|
+
const reflectedLineCount = stored.reflectedObservationLineCount ?? 0;
|
|
7870
|
+
const unreflectedContent = (stored.activeObservations || "").split("\n").slice(reflectedLineCount).join("\n").trim();
|
|
7871
|
+
const newObservations = unreflectedContent ? `${bufferedReflection}\n\n${unreflectedContent}` : bufferedReflection;
|
|
7872
|
+
const tokenCount = input.tokenCount + Math.max(0, (stored.observationTokenCount ?? 0) - (currentRecord.observationTokenCount ?? 0));
|
|
7873
|
+
return this.rollOverOMRow(connection, stored.id, newObservations, tokenCount, input.newRecordId, true);
|
|
7819
7874
|
});
|
|
7820
|
-
const nowSql = transformToSqlValue(/* @__PURE__ */ new Date());
|
|
7821
|
-
await this.pool.execute(`UPDATE ${OM_TABLE_QUOTED} SET
|
|
7822
|
-
${omCol("bufferedReflection")} = NULL,
|
|
7823
|
-
${omCol("bufferedReflectionTokens")} = NULL,
|
|
7824
|
-
${omCol("bufferedReflectionInputTokens")} = NULL,
|
|
7825
|
-
${omCol("reflectedObservationLineCount")} = NULL,
|
|
7826
|
-
${omCol("updatedAt")} = ?
|
|
7827
|
-
WHERE ${omCol("id")} = ?`, [nowSql, input.currentRecord.id]);
|
|
7828
|
-
return newRecord;
|
|
7829
7875
|
} catch (error) {
|
|
7830
7876
|
rethrowOrWrapOM(error, input.currentRecord.id, "SWAP_BUFFERED_REFLECTION_TO_ACTIVE");
|
|
7831
7877
|
}
|
|
7832
7878
|
}
|
|
7879
|
+
/**
|
|
7880
|
+
* Runs `fn` in one transaction on a dedicated connection. Every OM lifecycle write locks its
|
|
7881
|
+
* target row by primary key (`SELECT … FOR UPDATE`, a record lock — never a range), so the
|
|
7882
|
+
* only expected lock conflict is a wait; a deadlock or lock-wait timeout (e.g. against the
|
|
7883
|
+
* startup backfill's scan) is retried a bounded number of times.
|
|
7884
|
+
*/
|
|
7885
|
+
async withOMTransaction(fn) {
|
|
7886
|
+
for (let attempt = 1;; attempt++) {
|
|
7887
|
+
const connection = await this.pool.getConnection();
|
|
7888
|
+
try {
|
|
7889
|
+
await connection.beginTransaction();
|
|
7890
|
+
const result = await fn(connection);
|
|
7891
|
+
await connection.commit();
|
|
7892
|
+
return result;
|
|
7893
|
+
} catch (error) {
|
|
7894
|
+
await connection.rollback().catch(() => {});
|
|
7895
|
+
if (attempt < OM_MAX_TRANSACTION_ATTEMPTS && isRetryableLockError(error)) continue;
|
|
7896
|
+
throw error;
|
|
7897
|
+
} finally {
|
|
7898
|
+
connection.release();
|
|
7899
|
+
}
|
|
7900
|
+
}
|
|
7901
|
+
}
|
|
7902
|
+
/** Row-locks one OM record by primary key for the rest of the transaction. */
|
|
7903
|
+
async lockOMRow(connection, id) {
|
|
7904
|
+
const [rows] = await connection.execute(`SELECT * FROM ${OM_TABLE_QUOTED} WHERE ${omCol("id")} = ? FOR UPDATE`, [id]);
|
|
7905
|
+
return rows?.[0] ?? null;
|
|
7906
|
+
}
|
|
7907
|
+
/**
|
|
7908
|
+
* The live, row-locked record a lifecycle write aimed at `row` lands on: `row` itself while
|
|
7909
|
+
* live, otherwise the head. The head is found with a plain (non-locking, so no range locks)
|
|
7910
|
+
* canonical head read, then locked by primary key. Under REPEATABLE READ that read can predate
|
|
7911
|
+
* a rollover committed after this transaction started; the locking read sees the latest
|
|
7912
|
+
* committed row, so a head retired since is followed through `supersededBy` (bounded).
|
|
7913
|
+
*/
|
|
7914
|
+
async resolveLiveOMRow(connection, row) {
|
|
7915
|
+
if (!row.supersededBy) return row;
|
|
7916
|
+
const [heads] = await connection.execute(`SELECT id FROM ${OM_TABLE_QUOTED} WHERE ${omCol("lookupKey")} = ? ORDER BY ${OM_HEAD_ORDER} LIMIT 1`, [row.lookupKey]);
|
|
7917
|
+
const headId = heads?.[0]?.id ?? row.supersededBy;
|
|
7918
|
+
let current = await this.lockOMRow(connection, headId) ?? row;
|
|
7919
|
+
for (let hop = 0; current.supersededBy; hop++) {
|
|
7920
|
+
const next = hop < OM_MAX_HEAD_HOPS ? await this.lockOMRow(connection, current.supersededBy) : null;
|
|
7921
|
+
if (!next) throw new MastraError({
|
|
7922
|
+
id: createStorageErrorId("MYSQL", "RESOLVE_OBSERVATIONAL_MEMORY_HEAD", "FAILED"),
|
|
7923
|
+
text: `Observational memory record ${row.id} is superseded but no live head was found`,
|
|
7924
|
+
domain: ErrorDomain.STORAGE,
|
|
7925
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
7926
|
+
details: { id: row.id }
|
|
7927
|
+
});
|
|
7928
|
+
current = next;
|
|
7929
|
+
}
|
|
7930
|
+
return current;
|
|
7931
|
+
}
|
|
7932
|
+
/**
|
|
7933
|
+
* Create the next generation from the stored (live, row-locked) record and retire the stored
|
|
7934
|
+
* record in the same transaction. Carried columns are copied in SQL from the stored row, so
|
|
7935
|
+
* nothing round-trips through a client-side date conversion. Buffered chunks move to the new
|
|
7936
|
+
* generation; the cursor, buffering markers, flags, and counters carry over; buffered
|
|
7937
|
+
* reflection state does not (and is cleared on the retired row after a buffered swap).
|
|
7938
|
+
*/
|
|
7939
|
+
async rollOverOMRow(connection, storedId, observations, tokenCount, newRecordId, clearBufferedReflection) {
|
|
7940
|
+
const id = newRecordId ?? globalThis.crypto.randomUUID();
|
|
7941
|
+
const nowSql = transformToSqlValue(/* @__PURE__ */ new Date());
|
|
7942
|
+
const c = omCol;
|
|
7943
|
+
await connection.execute(`INSERT INTO ${OM_TABLE_QUOTED} (
|
|
7944
|
+
${c("id")}, ${c("lookupKey")}, ${c("scope")}, ${c("resourceId")}, ${c("threadId")},
|
|
7945
|
+
${c("activeObservations")}, ${c("activeObservationsPendingUpdate")},
|
|
7946
|
+
${c("originType")}, ${c("config")}, ${c("generationCount")}, ${c("lastObservedAt")}, ${c("lastReflectionAt")},
|
|
7947
|
+
${c("pendingMessageTokens")}, ${c("totalTokensObserved")}, ${c("observationTokenCount")}, ${c("bufferedObservationChunks")},
|
|
7948
|
+
${c("isObserving")}, ${c("isReflecting")}, ${c("isBufferingObservation")}, ${c("isBufferingReflection")},
|
|
7949
|
+
${c("lastBufferedAtTokens")}, ${c("lastBufferedAtTime")},
|
|
7950
|
+
${c("observedTimezone")}, ${c("metadata")}, ${c("supersededBy")}, ${c("createdAt")}, ${c("updatedAt")}
|
|
7951
|
+
)
|
|
7952
|
+
SELECT
|
|
7953
|
+
?, ${c("lookupKey")}, ${c("scope")}, ${c("resourceId")}, ${c("threadId")},
|
|
7954
|
+
?, NULL,
|
|
7955
|
+
'reflection', ${c("config")}, ${c("generationCount")} + 1, ${c("lastObservedAt")}, ?,
|
|
7956
|
+
${c("pendingMessageTokens")}, ${c("totalTokensObserved")}, ?, ${c("bufferedObservationChunks")},
|
|
7957
|
+
false, false, ${c("isBufferingObservation")}, false,
|
|
7958
|
+
${c("lastBufferedAtTokens")}, ${c("lastBufferedAtTime")},
|
|
7959
|
+
${c("observedTimezone")}, ${c("metadata")}, NULL, ?, ?
|
|
7960
|
+
FROM ${OM_TABLE_QUOTED} WHERE ${c("id")} = ?`, [
|
|
7961
|
+
id,
|
|
7962
|
+
observations,
|
|
7963
|
+
nowSql,
|
|
7964
|
+
Math.round(tokenCount),
|
|
7965
|
+
nowSql,
|
|
7966
|
+
nowSql,
|
|
7967
|
+
storedId
|
|
7968
|
+
]);
|
|
7969
|
+
const [retired] = await connection.execute(`UPDATE ${OM_TABLE_QUOTED} SET
|
|
7970
|
+
${c("supersededBy")} = ?,
|
|
7971
|
+
${c("bufferedObservationChunks")} = NULL,
|
|
7972
|
+
${clearBufferedReflection ? `${c("bufferedReflection")} = NULL, ${c("bufferedReflectionTokens")} = NULL,
|
|
7973
|
+
${c("bufferedReflectionInputTokens")} = NULL, ${c("reflectedObservationLineCount")} = NULL,` : ""}
|
|
7974
|
+
${c("updatedAt")} = ?
|
|
7975
|
+
WHERE ${c("id")} = ? AND ${c("supersededBy")} IS NULL`, [
|
|
7976
|
+
id,
|
|
7977
|
+
nowSql,
|
|
7978
|
+
storedId
|
|
7979
|
+
]);
|
|
7980
|
+
if (retired.affectedRows !== 1) throw new MastraError({
|
|
7981
|
+
id: createStorageErrorId("MYSQL", "CREATE_REFLECTION_GENERATION", "RETIRE_FAILED"),
|
|
7982
|
+
text: `Failed to retire observational memory record ${storedId}`,
|
|
7983
|
+
domain: ErrorDomain.STORAGE,
|
|
7984
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
7985
|
+
details: { id: storedId }
|
|
7986
|
+
});
|
|
7987
|
+
const inserted = await this.lockOMRow(connection, id);
|
|
7988
|
+
if (!inserted) throwOMNotFound(id, "CREATE_REFLECTION_GENERATION");
|
|
7989
|
+
return this.parseOMRow(inserted);
|
|
7990
|
+
}
|
|
7991
|
+
/**
|
|
7992
|
+
* Retire every live row that sorts after its key's canonical head, marking it superseded by
|
|
7993
|
+
* the head. Rows retired by older adapter versions (which never set `supersededBy`) become
|
|
7994
|
+
* frozen. Only keys with more than one live row are touched; each update embeds the head's
|
|
7995
|
+
* ordering key, so a row created by a concurrent rollover (a newer generation) never matches.
|
|
7996
|
+
* Runs at READ COMMITTED so the scan takes no gap locks.
|
|
7997
|
+
*/
|
|
7998
|
+
async backfillSupersededBy() {
|
|
7999
|
+
const [keys] = await this.pool.execute(`SELECT ${omCol("lookupKey")} AS lookupKey FROM ${OM_TABLE_QUOTED}
|
|
8000
|
+
WHERE ${omCol("supersededBy")} IS NULL GROUP BY ${omCol("lookupKey")} HAVING COUNT(*) > 1`);
|
|
8001
|
+
for (const { lookupKey } of keys ?? []) for (let attempt = 1;; attempt++) {
|
|
8002
|
+
const connection = await this.pool.getConnection();
|
|
8003
|
+
try {
|
|
8004
|
+
await connection.query("SET TRANSACTION ISOLATION LEVEL READ COMMITTED");
|
|
8005
|
+
await connection.beginTransaction();
|
|
8006
|
+
await connection.execute(`UPDATE ${OM_TABLE_QUOTED} AS r
|
|
8007
|
+
JOIN (
|
|
8008
|
+
SELECT ${omCol("id")} AS id, ${omCol("generationCount")} AS generationCount, ${omCol("createdAt")} AS createdAt
|
|
8009
|
+
FROM ${OM_TABLE_QUOTED} WHERE ${omCol("lookupKey")} = ? ORDER BY ${OM_HEAD_ORDER} LIMIT 1
|
|
8010
|
+
) AS h
|
|
8011
|
+
SET r.${omCol("supersededBy")} = h.id
|
|
8012
|
+
WHERE r.${omCol("lookupKey")} = ?
|
|
8013
|
+
AND r.${omCol("supersededBy")} IS NULL
|
|
8014
|
+
AND (
|
|
8015
|
+
r.${omCol("generationCount")} < h.generationCount
|
|
8016
|
+
OR (r.${omCol("generationCount")} = h.generationCount AND r.${omCol("createdAt")} > h.createdAt)
|
|
8017
|
+
OR (r.${omCol("generationCount")} = h.generationCount AND r.${omCol("createdAt")} = h.createdAt AND r.${omCol("id")} > h.id)
|
|
8018
|
+
)`, [lookupKey, lookupKey]);
|
|
8019
|
+
await connection.commit();
|
|
8020
|
+
break;
|
|
8021
|
+
} catch (error) {
|
|
8022
|
+
await connection.rollback().catch(() => {});
|
|
8023
|
+
if (attempt < OM_MAX_TRANSACTION_ATTEMPTS && isRetryableLockError(error)) continue;
|
|
8024
|
+
throw error;
|
|
8025
|
+
} finally {
|
|
8026
|
+
connection.release();
|
|
8027
|
+
}
|
|
8028
|
+
}
|
|
8029
|
+
}
|
|
7833
8030
|
async updateOMFlag(id, column, value, operation) {
|
|
7834
8031
|
try {
|
|
7835
8032
|
const [result] = await this.pool.execute(`UPDATE ${OM_TABLE_QUOTED} SET ${omCol(column)} = ?, ${omCol("updatedAt")} = ? WHERE ${omCol("id")} = ?`, [
|
|
@@ -7877,7 +8074,8 @@ var MemoryMySQL = class MemoryMySQL extends MemoryStorage {
|
|
|
7877
8074
|
config: parseJSONColumn(row.config) ?? {},
|
|
7878
8075
|
metadata: parseJSONColumn(row.metadata),
|
|
7879
8076
|
observedMessageIds: parseJSONColumn(row.observedMessageIds),
|
|
7880
|
-
observedTimezone: row.observedTimezone || void 0
|
|
8077
|
+
observedTimezone: row.observedTimezone || void 0,
|
|
8078
|
+
supersededBy: row.supersededBy ?? null
|
|
7881
8079
|
};
|
|
7882
8080
|
}
|
|
7883
8081
|
};
|