@mastra/pg 1.14.3 → 1.15.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/docs/SKILL.md +2 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/docs-memory-semantic-recall.md +8 -0
- package/dist/docs/references/docs-memory-storage.md +6 -0
- package/dist/docs/references/docs-memory-working-memory.md +2 -0
- package/dist/docs/references/docs-rag-overview.md +2 -0
- package/dist/docs/references/docs-rag-retrieval.md +2 -0
- package/dist/docs/references/docs-rag-vector-databases.md +2 -0
- package/dist/docs/references/reference-memory-memory-class.md +3 -0
- package/dist/docs/references/reference-processors-message-history-processor.md +3 -0
- package/dist/docs/references/reference-processors-semantic-recall-processor.md +3 -0
- package/dist/docs/references/reference-processors-working-memory-processor.md +3 -0
- package/dist/docs/references/reference-rag-metadata-filters.md +2 -0
- package/dist/docs/references/reference-storage-composite.md +2 -0
- package/dist/docs/references/reference-storage-dynamodb.md +2 -0
- package/dist/docs/references/reference-storage-postgresql.md +2 -0
- package/dist/docs/references/reference-storage-retention.md +180 -0
- package/dist/docs/references/reference-tools-vector-query-tool.md +2 -0
- package/dist/docs/references/reference-vectors-pg.md +2 -0
- package/dist/index.cjs +749 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +750 -3
- package/dist/index.js.map +1 -1
- package/dist/shared/config.d.ts +8 -1
- package/dist/shared/config.d.ts.map +1 -1
- package/dist/storage/db/index.d.ts +51 -0
- package/dist/storage/db/index.d.ts.map +1 -1
- package/dist/storage/domains/background-tasks/index.d.ts +19 -1
- package/dist/storage/domains/background-tasks/index.d.ts.map +1 -1
- package/dist/storage/domains/experiments/index.d.ts +31 -1
- package/dist/storage/domains/experiments/index.d.ts.map +1 -1
- package/dist/storage/domains/memory/index.d.ts +39 -1
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/notifications/index.d.ts +18 -1
- package/dist/storage/domains/notifications/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/index.d.ts +22 -1
- package/dist/storage/domains/observability/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +17 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/retention.d.ts +64 -0
- package/dist/storage/domains/observability/v-next/retention.d.ts.map +1 -0
- package/dist/storage/domains/schedules/index.d.ts +23 -1
- package/dist/storage/domains/schedules/index.d.ts.map +1 -1
- package/dist/storage/domains/scores/index.d.ts +18 -1
- package/dist/storage/domains/scores/index.d.ts.map +1 -1
- package/dist/storage/domains/workflows/index.d.ts +19 -1
- package/dist/storage/domains/workflows/index.d.ts.map +1 -1
- package/dist/storage/retention.d.ts +76 -0
- package/dist/storage/retention.d.ts.map +1 -0
- package/package.json +3 -3
package/dist/index.cjs
CHANGED
|
@@ -2542,6 +2542,11 @@ CREATE TRIGGER ${triggerName}
|
|
|
2542
2542
|
EXECUTE FUNCTION ${functionName}();`;
|
|
2543
2543
|
}
|
|
2544
2544
|
var schemaSetupRegistry = /* @__PURE__ */ new Map();
|
|
2545
|
+
function assertPositiveLimit(limit) {
|
|
2546
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
|
2547
|
+
throw new Error(`prune limit must be a positive integer; received ${limit}`);
|
|
2548
|
+
}
|
|
2549
|
+
}
|
|
2545
2550
|
var PgDB = class extends base.MastraBase {
|
|
2546
2551
|
client;
|
|
2547
2552
|
schemaName;
|
|
@@ -3578,6 +3583,96 @@ Note: This migration may take some time for large tables.
|
|
|
3578
3583
|
async deleteData({ tableName }) {
|
|
3579
3584
|
return this.clearTable({ tableName });
|
|
3580
3585
|
}
|
|
3586
|
+
// ---------------------------------------------------------------------------
|
|
3587
|
+
// Retention helpers (prune)
|
|
3588
|
+
// ---------------------------------------------------------------------------
|
|
3589
|
+
/**
|
|
3590
|
+
* Deletes up to `limit` rows from `tableName` whose `column` value is strictly
|
|
3591
|
+
* older than `cutoff`, in a single bounded statement. Returns the number of
|
|
3592
|
+
* rows deleted so the caller's batch loop can decide whether the table is
|
|
3593
|
+
* drained.
|
|
3594
|
+
*
|
|
3595
|
+
* PostgreSQL has no `DELETE ... LIMIT`, so this targets a bounded set of
|
|
3596
|
+
* physical rows via the `ctid` system column (PG's row identity, analogous to
|
|
3597
|
+
* SQLite's `rowid`). `cutoff` is bound as a parameter — a `Date`/ISO-8601
|
|
3598
|
+
* string compared against a `timestamptz` anchor column, or a `number`
|
|
3599
|
+
* compared against a `bigint` epoch-ms anchor column.
|
|
3600
|
+
*/
|
|
3601
|
+
async pruneBatch({
|
|
3602
|
+
tableName,
|
|
3603
|
+
column,
|
|
3604
|
+
cutoff,
|
|
3605
|
+
limit
|
|
3606
|
+
}) {
|
|
3607
|
+
assertPositiveLimit(limit);
|
|
3608
|
+
const fullTableName = getTableName({ indexName: tableName, schemaName: getSchemaName(this.schemaName) });
|
|
3609
|
+
const parsedColumn = `"${utils.parseSqlIdentifier(column, "column name")}"`;
|
|
3610
|
+
const sql = `
|
|
3611
|
+
DELETE FROM ${fullTableName}
|
|
3612
|
+
WHERE ctid IN (
|
|
3613
|
+
SELECT ctid FROM ${fullTableName}
|
|
3614
|
+
WHERE ${parsedColumn} < $1
|
|
3615
|
+
LIMIT $2
|
|
3616
|
+
)
|
|
3617
|
+
`;
|
|
3618
|
+
const result = await this.client.query(sql, [cutoff, limit]);
|
|
3619
|
+
return result.rowCount ?? 0;
|
|
3620
|
+
}
|
|
3621
|
+
/**
|
|
3622
|
+
* Deletes up to `limit` aged parent rows *and* their child rows together, in
|
|
3623
|
+
* a single transaction (used by whole-unit pruning such as experiments →
|
|
3624
|
+
* experiment_results). The aged parent IDs are selected first and both
|
|
3625
|
+
* deletes target that exact ID set, so a bound or abort between batches never
|
|
3626
|
+
* leaves a parent hollow (kept, but with its children gone) or children
|
|
3627
|
+
* orphaned.
|
|
3628
|
+
*/
|
|
3629
|
+
async pruneUnitsBatch({
|
|
3630
|
+
parentTable,
|
|
3631
|
+
parentKey,
|
|
3632
|
+
parentColumn,
|
|
3633
|
+
childTable,
|
|
3634
|
+
childForeignKey,
|
|
3635
|
+
cutoff,
|
|
3636
|
+
limit
|
|
3637
|
+
}) {
|
|
3638
|
+
assertPositiveLimit(limit);
|
|
3639
|
+
const schemaName = getSchemaName(this.schemaName);
|
|
3640
|
+
const fullChildTable = getTableName({ indexName: childTable, schemaName });
|
|
3641
|
+
const fullParentTable = getTableName({ indexName: parentTable, schemaName });
|
|
3642
|
+
const childFk = `"${utils.parseSqlIdentifier(childForeignKey, "column name")}"`;
|
|
3643
|
+
const parentPk = `"${utils.parseSqlIdentifier(parentKey, "column name")}"`;
|
|
3644
|
+
const parentCol = `"${utils.parseSqlIdentifier(parentColumn, "column name")}"`;
|
|
3645
|
+
return this.client.tx(async (t) => {
|
|
3646
|
+
const rows = await t.manyOrNone(
|
|
3647
|
+
`SELECT ${parentPk} AS id FROM ${fullParentTable} WHERE ${parentCol} < $1 LIMIT $2 FOR UPDATE SKIP LOCKED`,
|
|
3648
|
+
[cutoff, limit]
|
|
3649
|
+
);
|
|
3650
|
+
const ids = rows.map((r) => r.id);
|
|
3651
|
+
if (ids.length === 0) return { parents: 0, children: 0 };
|
|
3652
|
+
const childResult = await t.query(`DELETE FROM ${fullChildTable} WHERE ${childFk} = ANY($1)`, [ids]);
|
|
3653
|
+
const parentResult = await t.query(`DELETE FROM ${fullParentTable} WHERE ${parentPk} = ANY($1)`, [ids]);
|
|
3654
|
+
return {
|
|
3655
|
+
parents: parentResult.rowCount ?? 0,
|
|
3656
|
+
children: childResult.rowCount ?? 0
|
|
3657
|
+
};
|
|
3658
|
+
});
|
|
3659
|
+
}
|
|
3660
|
+
/**
|
|
3661
|
+
* Creates a btree index on `column` for `tableName` if it does not already
|
|
3662
|
+
* exist, so age-based prune deletes stay fast. Delegates to {@link createIndex}
|
|
3663
|
+
* (which is a no-op when the index is present).
|
|
3664
|
+
*
|
|
3665
|
+
* The name is lowercased and truncated to Postgres' 63-byte identifier limit
|
|
3666
|
+
* (schema-prefixed names can exceed it), mirroring {@link buildConstraintName}.
|
|
3667
|
+
*/
|
|
3668
|
+
async ensureIndex({
|
|
3669
|
+
indexName,
|
|
3670
|
+
tableName,
|
|
3671
|
+
column
|
|
3672
|
+
}) {
|
|
3673
|
+
const name = buildConstraintName({ baseName: indexName });
|
|
3674
|
+
await this.createIndex({ name, table: tableName, columns: [column] });
|
|
3675
|
+
}
|
|
3581
3676
|
async executeUpdate(client, {
|
|
3582
3677
|
tableName,
|
|
3583
3678
|
keys,
|
|
@@ -4536,6 +4631,98 @@ var AgentsPG = class _AgentsPG extends storage.AgentsStorage {
|
|
|
4536
4631
|
};
|
|
4537
4632
|
}
|
|
4538
4633
|
};
|
|
4634
|
+
var DEFAULT_BATCH_SIZE = 1e3;
|
|
4635
|
+
async function sleep(ms, signal) {
|
|
4636
|
+
if (ms <= 0 || signal?.aborted) return;
|
|
4637
|
+
await new Promise((resolve) => {
|
|
4638
|
+
const timer = setTimeout(() => {
|
|
4639
|
+
signal?.removeEventListener("abort", onAbort);
|
|
4640
|
+
resolve();
|
|
4641
|
+
}, ms);
|
|
4642
|
+
const onAbort = () => {
|
|
4643
|
+
clearTimeout(timer);
|
|
4644
|
+
resolve();
|
|
4645
|
+
};
|
|
4646
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
4647
|
+
});
|
|
4648
|
+
}
|
|
4649
|
+
function cutoffFor(policy, anchorType, now = Date.now()) {
|
|
4650
|
+
const cutoffMs = now - storage.parseDuration(policy.maxAge);
|
|
4651
|
+
return anchorType === "epoch-ms" ? cutoffMs : new Date(cutoffMs);
|
|
4652
|
+
}
|
|
4653
|
+
async function runBatchedDelete({
|
|
4654
|
+
deleteBatch,
|
|
4655
|
+
batchSize,
|
|
4656
|
+
options
|
|
4657
|
+
}) {
|
|
4658
|
+
if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {
|
|
4659
|
+
throw new Error(`retention batchSize must be a positive integer; received ${batchSize}`);
|
|
4660
|
+
}
|
|
4661
|
+
let deleted = 0;
|
|
4662
|
+
let batches = 0;
|
|
4663
|
+
while (true) {
|
|
4664
|
+
if (options?.signal?.aborted) return { deleted, done: false };
|
|
4665
|
+
if (options?.maxBatches !== void 0 && batches >= options.maxBatches) return { deleted, done: false };
|
|
4666
|
+
let limit = batchSize;
|
|
4667
|
+
if (options?.maxRows !== void 0) {
|
|
4668
|
+
const remaining = options.maxRows - deleted;
|
|
4669
|
+
if (remaining <= 0) return { deleted, done: false };
|
|
4670
|
+
limit = Math.min(limit, remaining);
|
|
4671
|
+
}
|
|
4672
|
+
const affected = await deleteBatch(limit);
|
|
4673
|
+
deleted += affected;
|
|
4674
|
+
batches += 1;
|
|
4675
|
+
if (affected < limit) return { deleted, done: true };
|
|
4676
|
+
if (options?.pauseMs) {
|
|
4677
|
+
await sleep(options.pauseMs, options.signal);
|
|
4678
|
+
}
|
|
4679
|
+
}
|
|
4680
|
+
}
|
|
4681
|
+
async function runPrune({
|
|
4682
|
+
db,
|
|
4683
|
+
domain,
|
|
4684
|
+
targets,
|
|
4685
|
+
options
|
|
4686
|
+
}) {
|
|
4687
|
+
const results = [];
|
|
4688
|
+
const now = Date.now();
|
|
4689
|
+
for (const target of targets) {
|
|
4690
|
+
if (options?.signal?.aborted) {
|
|
4691
|
+
results.push({ domain, table: target.table, deleted: 0, done: false });
|
|
4692
|
+
continue;
|
|
4693
|
+
}
|
|
4694
|
+
const cutoff = cutoffFor(target.policy, target.anchorType, now);
|
|
4695
|
+
const batchSize = target.policy.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
4696
|
+
const { deleted, done } = await runBatchedDelete({
|
|
4697
|
+
deleteBatch: (limit) => db.pruneBatch({ tableName: target.table, column: target.column, cutoff, limit }),
|
|
4698
|
+
batchSize,
|
|
4699
|
+
options
|
|
4700
|
+
});
|
|
4701
|
+
results.push({ domain, table: target.table, deleted, done });
|
|
4702
|
+
}
|
|
4703
|
+
return results;
|
|
4704
|
+
}
|
|
4705
|
+
function resolveTargets({
|
|
4706
|
+
policies,
|
|
4707
|
+
descriptor,
|
|
4708
|
+
order
|
|
4709
|
+
}) {
|
|
4710
|
+
const targets = [];
|
|
4711
|
+
for (const key of order) {
|
|
4712
|
+
const policy = policies[key];
|
|
4713
|
+
const entry = descriptor[key];
|
|
4714
|
+
if (!policy || !entry) continue;
|
|
4715
|
+
targets.push({
|
|
4716
|
+
table: entry.table,
|
|
4717
|
+
column: entry.column,
|
|
4718
|
+
anchorType: entry.anchorType ?? "timestamp",
|
|
4719
|
+
policy
|
|
4720
|
+
});
|
|
4721
|
+
}
|
|
4722
|
+
return targets;
|
|
4723
|
+
}
|
|
4724
|
+
|
|
4725
|
+
// src/storage/domains/background-tasks/index.ts
|
|
4539
4726
|
function getSchemaName3(schema) {
|
|
4540
4727
|
return schema ? `"${schema}"` : '"public"';
|
|
4541
4728
|
}
|
|
@@ -4590,6 +4777,14 @@ var BackgroundTasksPG = class _BackgroundTasksPG extends storage.BackgroundTasks
|
|
|
4590
4777
|
#skipDefaultIndexes;
|
|
4591
4778
|
#indexes;
|
|
4592
4779
|
static MANAGED_TABLES = [storage.TABLE_BACKGROUND_TASKS];
|
|
4780
|
+
/**
|
|
4781
|
+
* Completed background tasks accumulate as dead weight. Anchored on the
|
|
4782
|
+
* timezone-aware `completedAtZ` mirror column: NULL (in-flight) rows are
|
|
4783
|
+
* excluded automatically since `completedAtZ < cutoff` is false for NULL.
|
|
4784
|
+
*/
|
|
4785
|
+
static retentionTables = {
|
|
4786
|
+
backgroundTasks: { table: storage.TABLE_BACKGROUND_TASKS, column: "completedAtZ", indexed: true }
|
|
4787
|
+
};
|
|
4593
4788
|
constructor(config) {
|
|
4594
4789
|
super();
|
|
4595
4790
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -4611,6 +4806,40 @@ var BackgroundTasksPG = class _BackgroundTasksPG extends storage.BackgroundTasks
|
|
|
4611
4806
|
await this.createDefaultIndexes();
|
|
4612
4807
|
await this.createCustomIndexes();
|
|
4613
4808
|
}
|
|
4809
|
+
/**
|
|
4810
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
4811
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
4812
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
4813
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
4814
|
+
* logged and pruning proceeds (correct, just slower).
|
|
4815
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
4816
|
+
* so its supporting index is not part of the default index set.
|
|
4817
|
+
*/
|
|
4818
|
+
async ensureRetentionIndexes(policies) {
|
|
4819
|
+
const prefix = this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
4820
|
+
for (const [key, entry] of Object.entries(_BackgroundTasksPG.retentionTables)) {
|
|
4821
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
4822
|
+
try {
|
|
4823
|
+
await this.#db.ensureIndex({
|
|
4824
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
4825
|
+
tableName: entry.table,
|
|
4826
|
+
column: entry.column
|
|
4827
|
+
});
|
|
4828
|
+
} catch (error) {
|
|
4829
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
4830
|
+
}
|
|
4831
|
+
}
|
|
4832
|
+
}
|
|
4833
|
+
/** Delete completed tasks older than the `backgroundTasks` policy's `maxAge`, batched. */
|
|
4834
|
+
async prune(policies, options) {
|
|
4835
|
+
await this.ensureRetentionIndexes(policies);
|
|
4836
|
+
const targets = resolveTargets({
|
|
4837
|
+
policies,
|
|
4838
|
+
descriptor: _BackgroundTasksPG.retentionTables,
|
|
4839
|
+
order: ["backgroundTasks"]
|
|
4840
|
+
});
|
|
4841
|
+
return runPrune({ db: this.#db, domain: "backgroundTasks", targets, options });
|
|
4842
|
+
}
|
|
4614
4843
|
static getDefaultIndexDefs(schemaPrefix) {
|
|
4615
4844
|
return [
|
|
4616
4845
|
{
|
|
@@ -6248,12 +6477,24 @@ var DatasetsPG = class _DatasetsPG extends storage.DatasetsStorage {
|
|
|
6248
6477
|
await this.#db.clearTable({ tableName: storage.TABLE_DATASETS });
|
|
6249
6478
|
}
|
|
6250
6479
|
};
|
|
6480
|
+
var DEFAULT_PRUNE_BATCH_SIZE = 1e3;
|
|
6251
6481
|
var ExperimentsPG = class _ExperimentsPG extends storage.ExperimentsStorage {
|
|
6252
6482
|
#db;
|
|
6253
6483
|
#schema;
|
|
6254
6484
|
#skipDefaultIndexes;
|
|
6255
6485
|
#indexes;
|
|
6256
6486
|
static MANAGED_TABLES = [storage.TABLE_EXPERIMENTS, storage.TABLE_EXPERIMENT_RESULTS];
|
|
6487
|
+
/**
|
|
6488
|
+
* Experiments prune as whole units: an aged experiment and its result rows go
|
|
6489
|
+
* together, mirroring `deleteExperiment`. Anchored on `completedAt` (not the
|
|
6490
|
+
* `completedAtZ` mirror, which carries a `DEFAULT NOW()` this domain never
|
|
6491
|
+
* overrides — it holds insert time even for running rows). `completedAt` is
|
|
6492
|
+
* written as a UTC ISO string and stays NULL while running, so
|
|
6493
|
+
* `completedAt < cutoff` is false for in-flight experiments.
|
|
6494
|
+
*/
|
|
6495
|
+
static retentionTables = {
|
|
6496
|
+
experiments: { table: storage.TABLE_EXPERIMENTS, column: "completedAt", indexed: true }
|
|
6497
|
+
};
|
|
6257
6498
|
constructor(config) {
|
|
6258
6499
|
super();
|
|
6259
6500
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -6292,6 +6533,75 @@ var ExperimentsPG = class _ExperimentsPG extends storage.ExperimentsStorage {
|
|
|
6292
6533
|
await this.createDefaultIndexes();
|
|
6293
6534
|
await this.createCustomIndexes();
|
|
6294
6535
|
}
|
|
6536
|
+
/**
|
|
6537
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
6538
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
6539
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
6540
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
6541
|
+
* logged and pruning proceeds (correct, just slower).
|
|
6542
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
6543
|
+
* so its supporting index is not part of the default index set.
|
|
6544
|
+
*/
|
|
6545
|
+
async ensureRetentionIndexes(policies) {
|
|
6546
|
+
const prefix = this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
6547
|
+
for (const [key, entry] of Object.entries(_ExperimentsPG.retentionTables)) {
|
|
6548
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
6549
|
+
try {
|
|
6550
|
+
await this.#db.ensureIndex({
|
|
6551
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
6552
|
+
tableName: entry.table,
|
|
6553
|
+
column: entry.column
|
|
6554
|
+
});
|
|
6555
|
+
} catch (error) {
|
|
6556
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
6557
|
+
}
|
|
6558
|
+
}
|
|
6559
|
+
}
|
|
6560
|
+
/**
|
|
6561
|
+
* Delete experiments whose `completedAt` is older than the policy's `maxAge`.
|
|
6562
|
+
*
|
|
6563
|
+
* Each batch selects up to `batchSize` aged experiments and deletes their
|
|
6564
|
+
* `experiment_results` rows and the experiment rows in one transaction —
|
|
6565
|
+
* mirroring `deleteExperiment` — so hitting `maxBatches`/`maxRows` or the
|
|
6566
|
+
* abort signal between batches never leaves a run hollow (parent kept,
|
|
6567
|
+
* results gone). NULL `completedAt` (still running) is excluded by the
|
|
6568
|
+
* `< cutoff` predicate. Bounds count whole experiments, not rows.
|
|
6569
|
+
*/
|
|
6570
|
+
async prune(policies, options) {
|
|
6571
|
+
const policy = policies["experiments"];
|
|
6572
|
+
if (!policy || options?.signal?.aborted) {
|
|
6573
|
+
return policy ? [
|
|
6574
|
+
{ domain: "experiments", table: storage.TABLE_EXPERIMENT_RESULTS, deleted: 0, done: false },
|
|
6575
|
+
{ domain: "experiments", table: storage.TABLE_EXPERIMENTS, deleted: 0, done: false }
|
|
6576
|
+
] : [];
|
|
6577
|
+
}
|
|
6578
|
+
await this.ensureRetentionIndexes(policies);
|
|
6579
|
+
const rawCutoff = cutoffFor(policy, "timestamp");
|
|
6580
|
+
const cutoff = rawCutoff instanceof Date ? rawCutoff.toISOString() : rawCutoff;
|
|
6581
|
+
const batchSize = policy.batchSize ?? DEFAULT_PRUNE_BATCH_SIZE;
|
|
6582
|
+
let childDeleted = 0;
|
|
6583
|
+
const parent = await runBatchedDelete({
|
|
6584
|
+
deleteBatch: async (limit) => {
|
|
6585
|
+
const { parents, children } = await this.#db.pruneUnitsBatch({
|
|
6586
|
+
parentTable: storage.TABLE_EXPERIMENTS,
|
|
6587
|
+
parentKey: "id",
|
|
6588
|
+
parentColumn: "completedAt",
|
|
6589
|
+
childTable: storage.TABLE_EXPERIMENT_RESULTS,
|
|
6590
|
+
childForeignKey: "experimentId",
|
|
6591
|
+
cutoff,
|
|
6592
|
+
limit
|
|
6593
|
+
});
|
|
6594
|
+
childDeleted += children;
|
|
6595
|
+
return parents;
|
|
6596
|
+
},
|
|
6597
|
+
batchSize,
|
|
6598
|
+
options
|
|
6599
|
+
});
|
|
6600
|
+
return [
|
|
6601
|
+
{ domain: "experiments", table: storage.TABLE_EXPERIMENT_RESULTS, deleted: childDeleted, done: parent.done },
|
|
6602
|
+
{ domain: "experiments", table: storage.TABLE_EXPERIMENTS, deleted: parent.deleted, done: parent.done }
|
|
6603
|
+
];
|
|
6604
|
+
}
|
|
6295
6605
|
getDefaultIndexDefinitions() {
|
|
6296
6606
|
return [
|
|
6297
6607
|
{ name: "idx_experiments_datasetid", table: storage.TABLE_EXPERIMENTS, columns: ["datasetId"] },
|
|
@@ -8460,6 +8770,18 @@ function dedupeMessagesForSave(messages) {
|
|
|
8460
8770
|
}
|
|
8461
8771
|
var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
|
|
8462
8772
|
supportsObservationalMemory = true;
|
|
8773
|
+
/**
|
|
8774
|
+
* Retention-eligible tables. `threads`, `messages`, and `resources` all anchor
|
|
8775
|
+
* on the timezone-aware `createdAtZ` mirror column (kept in sync by triggers),
|
|
8776
|
+
* and are indexed for fast batched deletes. Cascade order is enforced in
|
|
8777
|
+
* `prune()` (children before threads), not here. Observational memory has no
|
|
8778
|
+
* timestamp anchor and is deliberately excluded.
|
|
8779
|
+
*/
|
|
8780
|
+
static retentionTables = {
|
|
8781
|
+
messages: { table: storage.TABLE_MESSAGES, column: "createdAtZ", indexed: true },
|
|
8782
|
+
resources: { table: storage.TABLE_RESOURCES, column: "createdAtZ", indexed: true },
|
|
8783
|
+
threads: { table: storage.TABLE_THREADS, column: "createdAtZ", indexed: true }
|
|
8784
|
+
};
|
|
8463
8785
|
#db;
|
|
8464
8786
|
#schema;
|
|
8465
8787
|
#skipDefaultIndexes;
|
|
@@ -8505,6 +8827,30 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
|
|
|
8505
8827
|
await this.createDefaultIndexes();
|
|
8506
8828
|
await this.createCustomIndexes();
|
|
8507
8829
|
}
|
|
8830
|
+
/**
|
|
8831
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
8832
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
8833
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
8834
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
8835
|
+
* logged and pruning proceeds (correct, just slower).
|
|
8836
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
8837
|
+
* so its supporting index is not part of the default index set.
|
|
8838
|
+
*/
|
|
8839
|
+
async ensureRetentionIndexes(policies) {
|
|
8840
|
+
const prefix = this.#schema && this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
8841
|
+
for (const [key, entry] of Object.entries(_MemoryPG.retentionTables)) {
|
|
8842
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
8843
|
+
try {
|
|
8844
|
+
await this.#db.ensureIndex({
|
|
8845
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
8846
|
+
tableName: entry.table,
|
|
8847
|
+
column: entry.column
|
|
8848
|
+
});
|
|
8849
|
+
} catch (error) {
|
|
8850
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
8851
|
+
}
|
|
8852
|
+
}
|
|
8853
|
+
}
|
|
8508
8854
|
/**
|
|
8509
8855
|
* Returns default index definitions for the memory domain tables.
|
|
8510
8856
|
* @param schemaPrefix - Prefix for index names (e.g. "my_schema_" or "")
|
|
@@ -8605,6 +8951,76 @@ var MemoryPG = class _MemoryPG extends storage.MemoryStorage {
|
|
|
8605
8951
|
await this.#db.clearTable({ tableName: storage.TABLE_THREADS });
|
|
8606
8952
|
await this.#db.clearTable({ tableName: storage.TABLE_RESOURCES });
|
|
8607
8953
|
}
|
|
8954
|
+
/**
|
|
8955
|
+
* Deletes rows older than the configured `maxAge` per table, in bounded,
|
|
8956
|
+
* batched, cancellable chunks. Tables are pruned children-first (messages and
|
|
8957
|
+
* resources before threads) since PostgreSQL has no FK cascade in this schema.
|
|
8958
|
+
* Unset tables are kept forever.
|
|
8959
|
+
*
|
|
8960
|
+
* When a `messages` policy is set, semantic-recall embeddings for pruned
|
|
8961
|
+
* messages are also swept from same-schema `memory_messages*` vector tables
|
|
8962
|
+
* (best-effort, mirroring `deleteThread`). Embeddings held in an external
|
|
8963
|
+
* vector store are out of reach and must be pruned by the operator.
|
|
8964
|
+
*/
|
|
8965
|
+
async prune(policies, options) {
|
|
8966
|
+
await this.ensureRetentionIndexes(policies);
|
|
8967
|
+
const targets = resolveTargets({
|
|
8968
|
+
policies,
|
|
8969
|
+
descriptor: _MemoryPG.retentionTables,
|
|
8970
|
+
order: ["messages", "resources", "threads"]
|
|
8971
|
+
});
|
|
8972
|
+
const results = await runPrune({ db: this.#db, domain: "memory", targets, options });
|
|
8973
|
+
if (policies["messages"]) {
|
|
8974
|
+
await this.pruneOrphanedVectorRows(policies["messages"], options);
|
|
8975
|
+
}
|
|
8976
|
+
return results;
|
|
8977
|
+
}
|
|
8978
|
+
/**
|
|
8979
|
+
* Best-effort sweep of semantic-recall vector rows whose source message no
|
|
8980
|
+
* longer exists (e.g. it was just pruned), so recall doesn't keep returning
|
|
8981
|
+
* embeddings that resolve to nothing. Only same-schema default vector tables
|
|
8982
|
+
* (`memory_messages*`) are covered — the same set `deleteThread` cleans up.
|
|
8983
|
+
* Failures are logged, never thrown: vector cleanup must not fail the prune.
|
|
8984
|
+
*/
|
|
8985
|
+
async pruneOrphanedVectorRows(policy, options) {
|
|
8986
|
+
try {
|
|
8987
|
+
const schemaName = this.#schema || "public";
|
|
8988
|
+
const vectorTables = await this.#db.client.manyOrNone(
|
|
8989
|
+
`
|
|
8990
|
+
SELECT tablename
|
|
8991
|
+
FROM pg_tables
|
|
8992
|
+
WHERE schemaname = $1
|
|
8993
|
+
AND (tablename = 'memory_messages' OR tablename LIKE 'memory_messages_%')
|
|
8994
|
+
`,
|
|
8995
|
+
[schemaName]
|
|
8996
|
+
);
|
|
8997
|
+
const messagesTable = getTableName4({ indexName: storage.TABLE_MESSAGES, schemaName: getSchemaName4(this.#schema) });
|
|
8998
|
+
for (const { tablename } of vectorTables) {
|
|
8999
|
+
const vectorTableName = getTableName4({ indexName: tablename, schemaName: getSchemaName4(this.#schema) });
|
|
9000
|
+
await runBatchedDelete({
|
|
9001
|
+
deleteBatch: async (limit) => {
|
|
9002
|
+
const result = await this.#db.client.query(
|
|
9003
|
+
`
|
|
9004
|
+
DELETE FROM ${vectorTableName}
|
|
9005
|
+
WHERE ctid IN (
|
|
9006
|
+
SELECT v.ctid FROM ${vectorTableName} v
|
|
9007
|
+
WHERE v.metadata->>'message_id' IS NOT NULL
|
|
9008
|
+
AND NOT EXISTS (SELECT 1 FROM ${messagesTable} m WHERE m.id = v.metadata->>'message_id')
|
|
9009
|
+
LIMIT $1
|
|
9010
|
+
)
|
|
9011
|
+
`,
|
|
9012
|
+
[limit]
|
|
9013
|
+
);
|
|
9014
|
+
return result.rowCount ?? 0;
|
|
9015
|
+
},
|
|
9016
|
+
batchSize: policy.batchSize ?? 1e3,
|
|
9017
|
+
options
|
|
9018
|
+
});
|
|
9019
|
+
}
|
|
9020
|
+
} catch (error) {
|
|
9021
|
+
this.logger?.warn?.("Failed to sweep orphaned semantic-recall vector rows after prune:", error);
|
|
9022
|
+
}
|
|
9023
|
+
}
|
|
8608
9024
|
/**
|
|
8609
9025
|
* Normalizes message row from database by applying createdAtZ fallback
|
|
8610
9026
|
*/
|
|
@@ -10916,6 +11332,13 @@ var NotificationsPG = class _NotificationsPG extends storage.NotificationsStorag
|
|
|
10916
11332
|
#skipDefaultIndexes;
|
|
10917
11333
|
#indexes;
|
|
10918
11334
|
static MANAGED_TABLES = [storage.TABLE_NOTIFICATIONS];
|
|
11335
|
+
/**
|
|
11336
|
+
* Notifications are an append-only event feed that grows unbounded. Single
|
|
11337
|
+
* table, anchored on the timezone-aware `createdAtZ` mirror column.
|
|
11338
|
+
*/
|
|
11339
|
+
static retentionTables = {
|
|
11340
|
+
notifications: { table: storage.TABLE_NOTIFICATIONS, column: "createdAtZ", indexed: true }
|
|
11341
|
+
};
|
|
10919
11342
|
constructor(config) {
|
|
10920
11343
|
super();
|
|
10921
11344
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -10932,6 +11355,40 @@ var NotificationsPG = class _NotificationsPG extends storage.NotificationsStorag
|
|
|
10932
11355
|
await this.createDefaultIndexes();
|
|
10933
11356
|
await this.createCustomIndexes();
|
|
10934
11357
|
}
|
|
11358
|
+
/**
|
|
11359
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
11360
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
11361
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
11362
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
11363
|
+
* logged and pruning proceeds (correct, just slower).
|
|
11364
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
11365
|
+
* so its supporting index is not part of the default index set.
|
|
11366
|
+
*/
|
|
11367
|
+
async ensureRetentionIndexes(policies) {
|
|
11368
|
+
const prefix = this.#schema && this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
11369
|
+
for (const [key, entry] of Object.entries(_NotificationsPG.retentionTables)) {
|
|
11370
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
11371
|
+
try {
|
|
11372
|
+
await this.#db.ensureIndex({
|
|
11373
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
11374
|
+
tableName: entry.table,
|
|
11375
|
+
column: entry.column
|
|
11376
|
+
});
|
|
11377
|
+
} catch (error) {
|
|
11378
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
11379
|
+
}
|
|
11380
|
+
}
|
|
11381
|
+
}
|
|
11382
|
+
/** Delete notifications older than the `notifications` policy's `maxAge`, batched. */
|
|
11383
|
+
async prune(policies, options) {
|
|
11384
|
+
await this.ensureRetentionIndexes(policies);
|
|
11385
|
+
const targets = resolveTargets({
|
|
11386
|
+
policies,
|
|
11387
|
+
descriptor: _NotificationsPG.retentionTables,
|
|
11388
|
+
order: ["notifications"]
|
|
11389
|
+
});
|
|
11390
|
+
return runPrune({ db: this.#db, domain: "notifications", targets, options });
|
|
11391
|
+
}
|
|
10935
11392
|
static getDefaultIndexDefs(schemaPrefix) {
|
|
10936
11393
|
return [
|
|
10937
11394
|
{
|
|
@@ -11173,6 +11630,14 @@ var NotificationsPG = class _NotificationsPG extends storage.NotificationsStorag
|
|
|
11173
11630
|
}
|
|
11174
11631
|
};
|
|
11175
11632
|
var ObservabilityPG = class _ObservabilityPG extends storage.ObservabilityStorage {
|
|
11633
|
+
/**
|
|
11634
|
+
* Retention-eligible tables. The observability domain has a single physical
|
|
11635
|
+
* table (`mastra_ai_spans`); spans anchor on the timezone-aware `startedAtZ`
|
|
11636
|
+
* mirror column and are indexed for fast batched deletes.
|
|
11637
|
+
*/
|
|
11638
|
+
static retentionTables = {
|
|
11639
|
+
spans: { table: storage.TABLE_SPANS, column: "startedAtZ", indexed: true }
|
|
11640
|
+
};
|
|
11176
11641
|
#db;
|
|
11177
11642
|
#schema;
|
|
11178
11643
|
#skipDefaultIndexes;
|
|
@@ -11197,6 +11662,30 @@ var ObservabilityPG = class _ObservabilityPG extends storage.ObservabilityStorag
|
|
|
11197
11662
|
await this.createDefaultIndexes();
|
|
11198
11663
|
await this.createCustomIndexes();
|
|
11199
11664
|
}
|
|
11665
|
+
/**
|
|
11666
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
11667
|
+
* anchor column so age-based `prune()` deletes stay fast on large span tables.
|
|
11668
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
11669
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
11670
|
+
* logged and pruning proceeds (correct, just slower).
|
|
11671
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
11672
|
+
* so its supporting index is not part of the default index set.
|
|
11673
|
+
*/
|
|
11674
|
+
async ensureRetentionIndexes(policies) {
|
|
11675
|
+
const prefix = this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
11676
|
+
for (const [key, entry] of Object.entries(_ObservabilityPG.retentionTables)) {
|
|
11677
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
11678
|
+
try {
|
|
11679
|
+
await this.#db.ensureIndex({
|
|
11680
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
11681
|
+
tableName: entry.table,
|
|
11682
|
+
column: entry.column
|
|
11683
|
+
});
|
|
11684
|
+
} catch (error) {
|
|
11685
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
11686
|
+
}
|
|
11687
|
+
}
|
|
11688
|
+
}
|
|
11200
11689
|
/**
|
|
11201
11690
|
* Returns default index definitions for the observability domain tables.
|
|
11202
11691
|
* @param schemaPrefix - Prefix for index names (e.g. "my_schema_" or "")
|
|
@@ -11341,6 +11830,19 @@ var ObservabilityPG = class _ObservabilityPG extends storage.ObservabilityStorag
|
|
|
11341
11830
|
async dangerouslyClearAll() {
|
|
11342
11831
|
await this.#db.clearTable({ tableName: storage.TABLE_SPANS });
|
|
11343
11832
|
}
|
|
11833
|
+
/**
|
|
11834
|
+
* Deletes spans older than the configured `maxAge`, in bounded, batched,
|
|
11835
|
+
* cancellable chunks. Unset = keep forever.
|
|
11836
|
+
*/
|
|
11837
|
+
async prune(policies, options) {
|
|
11838
|
+
await this.ensureRetentionIndexes(policies);
|
|
11839
|
+
const targets = resolveTargets({
|
|
11840
|
+
policies,
|
|
11841
|
+
descriptor: _ObservabilityPG.retentionTables,
|
|
11842
|
+
order: ["spans"]
|
|
11843
|
+
});
|
|
11844
|
+
return runPrune({ db: this.#db, domain: "observability", targets, options });
|
|
11845
|
+
}
|
|
11344
11846
|
get tracingStrategy() {
|
|
11345
11847
|
return {
|
|
11346
11848
|
preferred: "batch-with-updates",
|
|
@@ -14037,6 +14539,78 @@ async function setupPartitioning(client, schema, options = {}) {
|
|
|
14037
14539
|
return "native";
|
|
14038
14540
|
}
|
|
14039
14541
|
}
|
|
14542
|
+
function retentionCutoff(policy, now = Date.now()) {
|
|
14543
|
+
return new Date(now - storage.parseDuration(policy.maxAge));
|
|
14544
|
+
}
|
|
14545
|
+
async function listChildPartitions(client, schema, table) {
|
|
14546
|
+
const rows = await client.manyOrNone(
|
|
14547
|
+
`SELECT c.relname AS "name", pg_get_expr(c.relpartbound, c.oid) AS "bound"
|
|
14548
|
+
FROM pg_inherits i
|
|
14549
|
+
JOIN pg_class c ON c.oid = i.inhrelid
|
|
14550
|
+
JOIN pg_class p ON p.oid = i.inhparent
|
|
14551
|
+
JOIN pg_namespace n ON n.oid = p.relnamespace
|
|
14552
|
+
WHERE n.nspname = $1 AND p.relname = $2`,
|
|
14553
|
+
[schema, table]
|
|
14554
|
+
);
|
|
14555
|
+
const partitions = [];
|
|
14556
|
+
for (const row of rows ?? []) {
|
|
14557
|
+
const match = /TO \('([^']+)'\)/.exec(row.bound ?? "");
|
|
14558
|
+
if (!match?.[1]) continue;
|
|
14559
|
+
const upperBound = new Date(match[1]);
|
|
14560
|
+
if (Number.isNaN(upperBound.getTime())) continue;
|
|
14561
|
+
partitions.push({ name: row.name, upperBound });
|
|
14562
|
+
}
|
|
14563
|
+
return partitions.sort((a, b) => a.upperBound.getTime() - b.upperBound.getTime());
|
|
14564
|
+
}
|
|
14565
|
+
async function prunePartitionedTable({
|
|
14566
|
+
client,
|
|
14567
|
+
schema,
|
|
14568
|
+
table,
|
|
14569
|
+
cutoff,
|
|
14570
|
+
options
|
|
14571
|
+
}) {
|
|
14572
|
+
const partitions = await listChildPartitions(client, schema, table);
|
|
14573
|
+
const droppable = partitions.filter((p) => p.upperBound.getTime() <= cutoff.getTime());
|
|
14574
|
+
let deleted = 0;
|
|
14575
|
+
let batches = 0;
|
|
14576
|
+
for (const partition of droppable) {
|
|
14577
|
+
if (options?.signal?.aborted) return { deleted, done: false };
|
|
14578
|
+
if (options?.maxBatches !== void 0 && batches >= options.maxBatches) return { deleted, done: false };
|
|
14579
|
+
if (options?.maxRows !== void 0 && deleted >= options.maxRows) return { deleted, done: false };
|
|
14580
|
+
const child = qualifiedName(schema, partition.name);
|
|
14581
|
+
const parent = qualifiedTable(schema, table);
|
|
14582
|
+
const row = await client.one(`SELECT count(*)::int AS "n" FROM ${child}`);
|
|
14583
|
+
await client.none(`ALTER TABLE ${parent} DETACH PARTITION ${child}`);
|
|
14584
|
+
await client.none(`DROP TABLE IF EXISTS ${child}`);
|
|
14585
|
+
deleted += row.n;
|
|
14586
|
+
batches += 1;
|
|
14587
|
+
}
|
|
14588
|
+
return { deleted, done: true };
|
|
14589
|
+
}
|
|
14590
|
+
async function pruneTimescaleTable({
|
|
14591
|
+
client,
|
|
14592
|
+
schema,
|
|
14593
|
+
table,
|
|
14594
|
+
cutoff,
|
|
14595
|
+
options
|
|
14596
|
+
}) {
|
|
14597
|
+
if (options?.signal?.aborted || options?.maxBatches === 0) return { deleted: 0, done: false };
|
|
14598
|
+
const tableExpr = qualifiedTable(schema, table);
|
|
14599
|
+
const chunks = await client.manyOrNone(
|
|
14600
|
+
`SELECT show_chunks($1::regclass, older_than => $2::timestamptz)::text AS "chunk"`,
|
|
14601
|
+
[tableExpr, cutoff.toISOString()]
|
|
14602
|
+
);
|
|
14603
|
+
let deleted = 0;
|
|
14604
|
+
for (const { chunk } of chunks ?? []) {
|
|
14605
|
+
const row = await client.one(`SELECT count(*)::int AS "n" FROM ${chunk}`);
|
|
14606
|
+
deleted += row.n;
|
|
14607
|
+
}
|
|
14608
|
+
await client.none(`SELECT drop_chunks($1::regclass, older_than => $2::timestamptz)`, [
|
|
14609
|
+
tableExpr,
|
|
14610
|
+
cutoff.toISOString()
|
|
14611
|
+
]);
|
|
14612
|
+
return { deleted, done: true };
|
|
14613
|
+
}
|
|
14040
14614
|
function applyScoreFilters(acc, filters) {
|
|
14041
14615
|
applyCommonFilters(acc, filters);
|
|
14042
14616
|
applySingleOrArrayFilter(acc, "scorerId", filters?.scorerId);
|
|
@@ -14783,7 +15357,7 @@ function wrapError(op, error$1, details) {
|
|
|
14783
15357
|
error$1
|
|
14784
15358
|
);
|
|
14785
15359
|
}
|
|
14786
|
-
var ObservabilityStoragePostgresVNext = class extends storage.ObservabilityStorage {
|
|
15360
|
+
var ObservabilityStoragePostgresVNext = class _ObservabilityStoragePostgresVNext extends storage.ObservabilityStorage {
|
|
14787
15361
|
#client;
|
|
14788
15362
|
#schema;
|
|
14789
15363
|
#partitioning;
|
|
@@ -14867,6 +15441,50 @@ var ObservabilityStoragePostgresVNext = class extends storage.ObservabilityStora
|
|
|
14867
15441
|
get partitionMode() {
|
|
14868
15442
|
return this.#partitionMode;
|
|
14869
15443
|
}
|
|
15444
|
+
// -------------------------------------------------------------------------
|
|
15445
|
+
// Retention
|
|
15446
|
+
// -------------------------------------------------------------------------
|
|
15447
|
+
/**
|
|
15448
|
+
* All five signal tables are insert-only growth tables. The anchor column is
|
|
15449
|
+
* each table's partition / chunk key, so age-based expiry drops whole day
|
|
15450
|
+
* partitions instead of deleting rows (see `./retention.ts`). `indexed: true`
|
|
15451
|
+
* reflects that expiry never scans — it acts on partition bounds.
|
|
15452
|
+
*/
|
|
15453
|
+
static retentionTables = {
|
|
15454
|
+
spans: { table: TABLE_SPAN_EVENTS, column: "endedAt", indexed: true },
|
|
15455
|
+
metrics: { table: TABLE_METRIC_EVENTS, column: "timestamp", indexed: true },
|
|
15456
|
+
logs: { table: TABLE_LOG_EVENTS, column: "timestamp", indexed: true },
|
|
15457
|
+
scores: { table: TABLE_SCORE_EVENTS, column: "timestamp", indexed: true },
|
|
15458
|
+
feedback: { table: TABLE_FEEDBACK_EVENTS, column: "timestamp", indexed: true }
|
|
15459
|
+
};
|
|
15460
|
+
/**
|
|
15461
|
+
* Expire signal events older than each table's `maxAge` by dropping whole
|
|
15462
|
+
* day partitions (native / pg_partman) or chunks (Timescale). Only
|
|
15463
|
+
* partitions wholly older than the cutoff are dropped, so the effective
|
|
15464
|
+
* granularity is one day. Each partition drop counts as one batch for
|
|
15465
|
+
* `maxBatches` / `maxRows` / abort-signal purposes; `deleted` reports the
|
|
15466
|
+
* row count of the dropped partitions.
|
|
15467
|
+
*/
|
|
15468
|
+
async prune(policies, options) {
|
|
15469
|
+
return this.#run("VNEXT_PRUNE", async () => {
|
|
15470
|
+
const mode = this.#partitionMode ??= await resolveMode(this.#client, this.#partitioning);
|
|
15471
|
+
const results = [];
|
|
15472
|
+
const now = Date.now();
|
|
15473
|
+
for (const [key, entry] of Object.entries(_ObservabilityStoragePostgresVNext.retentionTables)) {
|
|
15474
|
+
const policy = policies[key];
|
|
15475
|
+
if (!policy) continue;
|
|
15476
|
+
if (options?.signal?.aborted) {
|
|
15477
|
+
results.push({ domain: "observability", table: entry.table, deleted: 0, done: false });
|
|
15478
|
+
continue;
|
|
15479
|
+
}
|
|
15480
|
+
const cutoff = retentionCutoff(policy, now);
|
|
15481
|
+
const args = { client: this.#client, schema: this.#schema, table: entry.table, cutoff, options };
|
|
15482
|
+
const outcome = mode === "timescale" ? await pruneTimescaleTable(args) : await prunePartitionedTable(args);
|
|
15483
|
+
results.push({ domain: "observability", table: entry.table, deleted: outcome.deleted, done: outcome.done });
|
|
15484
|
+
}
|
|
15485
|
+
return results;
|
|
15486
|
+
});
|
|
15487
|
+
}
|
|
14870
15488
|
get observabilityStrategy() {
|
|
14871
15489
|
return { preferred: "insert-only", supported: ["insert-only"] };
|
|
14872
15490
|
}
|
|
@@ -15811,6 +16429,15 @@ var SchedulesPG = class _SchedulesPG extends storage.SchedulesStorage {
|
|
|
15811
16429
|
#indexes;
|
|
15812
16430
|
/** Tables managed by this domain */
|
|
15813
16431
|
static MANAGED_TABLES = [storage.TABLE_SCHEDULES, storage.TABLE_SCHEDULE_TRIGGERS];
|
|
16432
|
+
/**
|
|
16433
|
+
* The fire/run history (`schedule_triggers`, one row per fire) is the growth
|
|
16434
|
+
* table; schedule definitions are config and excluded. Anchored on
|
|
16435
|
+
* `actual_fire_at`, a bigint epoch-ms column (numeric comparison, not
|
|
16436
|
+
* timestamptz).
|
|
16437
|
+
*/
|
|
16438
|
+
static retentionTables = {
|
|
16439
|
+
triggers: { table: storage.TABLE_SCHEDULE_TRIGGERS, column: "actual_fire_at", indexed: true, anchorType: "epoch-ms" }
|
|
16440
|
+
};
|
|
15814
16441
|
constructor(config) {
|
|
15815
16442
|
super();
|
|
15816
16443
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -15832,6 +16459,43 @@ var SchedulesPG = class _SchedulesPG extends storage.SchedulesStorage {
|
|
|
15832
16459
|
await this.createDefaultIndexes();
|
|
15833
16460
|
await this.createCustomIndexes();
|
|
15834
16461
|
}
|
|
16462
|
+
/**
|
|
16463
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
16464
|
+
* anchor column so age-based `prune()` deletes stay fast. The default
|
|
16465
|
+
* composite index leads with `schedule_id`, so a bare `actual_fire_at` range
|
|
16466
|
+
* scan can't use it. Called from the prune path (not init) so only
|
|
16467
|
+
* deployments that configure retention pay the index's write/disk overhead.
|
|
16468
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
16469
|
+
* so its supporting index is not part of the default index set.
|
|
16470
|
+
*/
|
|
16471
|
+
async ensureRetentionIndexes(policies) {
|
|
16472
|
+
const prefix = this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
16473
|
+
for (const [key, entry] of Object.entries(_SchedulesPG.retentionTables)) {
|
|
16474
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
16475
|
+
try {
|
|
16476
|
+
await this.#db.ensureIndex({
|
|
16477
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
16478
|
+
tableName: entry.table,
|
|
16479
|
+
column: entry.column
|
|
16480
|
+
});
|
|
16481
|
+
} catch (error) {
|
|
16482
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
16483
|
+
}
|
|
16484
|
+
}
|
|
16485
|
+
}
|
|
16486
|
+
/**
|
|
16487
|
+
* Delete trigger (fire history) rows whose `actual_fire_at` is older than the
|
|
16488
|
+
* `triggers` policy's `maxAge`, batched. Schedule definitions are never pruned.
|
|
16489
|
+
*/
|
|
16490
|
+
async prune(policies, options) {
|
|
16491
|
+
await this.ensureRetentionIndexes(policies);
|
|
16492
|
+
const targets = resolveTargets({
|
|
16493
|
+
policies,
|
|
16494
|
+
descriptor: _SchedulesPG.retentionTables,
|
|
16495
|
+
order: ["triggers"]
|
|
16496
|
+
});
|
|
16497
|
+
return runPrune({ db: this.#db, domain: "schedules", targets, options });
|
|
16498
|
+
}
|
|
15835
16499
|
/**
|
|
15836
16500
|
* Returns default index definitions for the schedules domain.
|
|
15837
16501
|
* @param schemaPrefix - Prefix for index names (e.g. "my_schema_" or "")
|
|
@@ -16762,6 +17426,13 @@ var ScoresPG = class _ScoresPG extends storage.ScoresStorage {
|
|
|
16762
17426
|
#indexes;
|
|
16763
17427
|
/** Tables managed by this domain */
|
|
16764
17428
|
static MANAGED_TABLES = [storage.TABLE_SCORERS];
|
|
17429
|
+
/**
|
|
17430
|
+
* Scorer results accumulate as evals run. Single table, anchored on the
|
|
17431
|
+
* timezone-aware `createdAtZ` mirror column (kept in sync by triggers).
|
|
17432
|
+
*/
|
|
17433
|
+
static retentionTables = {
|
|
17434
|
+
scorers: { table: storage.TABLE_SCORERS, column: "createdAtZ", indexed: true }
|
|
17435
|
+
};
|
|
16765
17436
|
constructor(config) {
|
|
16766
17437
|
super();
|
|
16767
17438
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -16780,6 +17451,30 @@ var ScoresPG = class _ScoresPG extends storage.ScoresStorage {
|
|
|
16780
17451
|
await this.createDefaultIndexes();
|
|
16781
17452
|
await this.createCustomIndexes();
|
|
16782
17453
|
}
|
|
17454
|
+
/**
|
|
17455
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
17456
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
17457
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
17458
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
17459
|
+
* logged and pruning proceeds (correct, just slower).
|
|
17460
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
17461
|
+
* so its supporting index is not part of the default index set.
|
|
17462
|
+
*/
|
|
17463
|
+
async ensureRetentionIndexes(policies) {
|
|
17464
|
+
const prefix = this.#schema && this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
17465
|
+
for (const [key, entry] of Object.entries(_ScoresPG.retentionTables)) {
|
|
17466
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
17467
|
+
try {
|
|
17468
|
+
await this.#db.ensureIndex({
|
|
17469
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
17470
|
+
tableName: entry.table,
|
|
17471
|
+
column: entry.column
|
|
17472
|
+
});
|
|
17473
|
+
} catch (error) {
|
|
17474
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
17475
|
+
}
|
|
17476
|
+
}
|
|
17477
|
+
}
|
|
16783
17478
|
/**
|
|
16784
17479
|
* Returns default index definitions for the scores domain tables.
|
|
16785
17480
|
* @param schemaPrefix - Prefix for index names (e.g. "my_schema_" or "")
|
|
@@ -16854,6 +17549,16 @@ var ScoresPG = class _ScoresPG extends storage.ScoresStorage {
|
|
|
16854
17549
|
async dangerouslyClearAll() {
|
|
16855
17550
|
await this.#db.clearTable({ tableName: storage.TABLE_SCORERS });
|
|
16856
17551
|
}
|
|
17552
|
+
/** Delete scorer results older than the `scorers` policy's `maxAge`, batched. */
|
|
17553
|
+
async prune(policies, options) {
|
|
17554
|
+
await this.ensureRetentionIndexes(policies);
|
|
17555
|
+
const targets = resolveTargets({
|
|
17556
|
+
policies,
|
|
17557
|
+
descriptor: _ScoresPG.retentionTables,
|
|
17558
|
+
order: ["scorers"]
|
|
17559
|
+
});
|
|
17560
|
+
return runPrune({ db: this.#db, domain: "scores", targets, options });
|
|
17561
|
+
}
|
|
16857
17562
|
async getScoreById({ id }) {
|
|
16858
17563
|
try {
|
|
16859
17564
|
const result = await this.#db.client.oneOrNone(
|
|
@@ -18175,6 +18880,14 @@ var WorkflowsPG = class _WorkflowsPG extends storage.WorkflowsStorage {
|
|
|
18175
18880
|
#indexes;
|
|
18176
18881
|
/** Tables managed by this domain */
|
|
18177
18882
|
static MANAGED_TABLES = [storage.TABLE_WORKFLOW_SNAPSHOT];
|
|
18883
|
+
/**
|
|
18884
|
+
* Workflow run snapshots accumulate as runs execute. Anchored on the
|
|
18885
|
+
* timezone-aware `updatedAtZ` mirror column (last activity) so suspended or
|
|
18886
|
+
* long-running runs are not pruned by start age.
|
|
18887
|
+
*/
|
|
18888
|
+
static retentionTables = {
|
|
18889
|
+
workflowSnapshot: { table: storage.TABLE_WORKFLOW_SNAPSHOT, column: "updatedAtZ", indexed: true }
|
|
18890
|
+
};
|
|
18178
18891
|
constructor(config) {
|
|
18179
18892
|
super();
|
|
18180
18893
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -18246,6 +18959,40 @@ var WorkflowsPG = class _WorkflowsPG extends storage.WorkflowsStorage {
|
|
|
18246
18959
|
await this.createDefaultIndexes();
|
|
18247
18960
|
await this.createCustomIndexes();
|
|
18248
18961
|
}
|
|
18962
|
+
/**
|
|
18963
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
18964
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
18965
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
18966
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
18967
|
+
* logged and pruning proceeds (correct, just slower).
|
|
18968
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
18969
|
+
* so its supporting index is not part of the default index set.
|
|
18970
|
+
*/
|
|
18971
|
+
async ensureRetentionIndexes(policies) {
|
|
18972
|
+
const prefix = this.#schema && this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
18973
|
+
for (const [key, entry] of Object.entries(_WorkflowsPG.retentionTables)) {
|
|
18974
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
18975
|
+
try {
|
|
18976
|
+
await this.#db.ensureIndex({
|
|
18977
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
18978
|
+
tableName: entry.table,
|
|
18979
|
+
column: entry.column
|
|
18980
|
+
});
|
|
18981
|
+
} catch (error) {
|
|
18982
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
18983
|
+
}
|
|
18984
|
+
}
|
|
18985
|
+
}
|
|
18986
|
+
/** Delete workflow run snapshots older than the `workflowSnapshot` policy's `maxAge`, batched. */
|
|
18987
|
+
async prune(policies, options) {
|
|
18988
|
+
await this.ensureRetentionIndexes(policies);
|
|
18989
|
+
const targets = resolveTargets({
|
|
18990
|
+
policies,
|
|
18991
|
+
descriptor: _WorkflowsPG.retentionTables,
|
|
18992
|
+
order: ["workflowSnapshot"]
|
|
18993
|
+
});
|
|
18994
|
+
return runPrune({ db: this.#db, domain: "workflows", targets, options });
|
|
18995
|
+
}
|
|
18249
18996
|
/**
|
|
18250
18997
|
* Creates custom user-defined indexes for this domain's tables.
|
|
18251
18998
|
*/
|
|
@@ -19359,7 +20106,7 @@ var PostgresStore = class extends storage.MastraCompositeStore {
|
|
|
19359
20106
|
constructor(config) {
|
|
19360
20107
|
try {
|
|
19361
20108
|
validateConfig("PostgresStore", config);
|
|
19362
|
-
super({ id: config.id, name: "PostgresStore", disableInit: config.disableInit });
|
|
20109
|
+
super({ id: config.id, name: "PostgresStore", disableInit: config.disableInit, retention: config.retention });
|
|
19363
20110
|
this.schema = utils.parseSqlIdentifier(config.schemaName || "public", "schema name");
|
|
19364
20111
|
if (isPoolConfig(config)) {
|
|
19365
20112
|
this.#pool = config.pool;
|