@mastra/pg 1.14.3 → 1.15.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/docs/SKILL.md +2 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/docs-memory-semantic-recall.md +8 -0
- package/dist/docs/references/docs-memory-storage.md +6 -0
- package/dist/docs/references/docs-memory-working-memory.md +2 -0
- package/dist/docs/references/docs-rag-overview.md +2 -0
- package/dist/docs/references/docs-rag-retrieval.md +2 -0
- package/dist/docs/references/docs-rag-vector-databases.md +2 -0
- package/dist/docs/references/reference-memory-memory-class.md +3 -0
- package/dist/docs/references/reference-processors-message-history-processor.md +3 -0
- package/dist/docs/references/reference-processors-semantic-recall-processor.md +3 -0
- package/dist/docs/references/reference-processors-working-memory-processor.md +3 -0
- package/dist/docs/references/reference-rag-metadata-filters.md +2 -0
- package/dist/docs/references/reference-storage-composite.md +2 -0
- package/dist/docs/references/reference-storage-dynamodb.md +2 -0
- package/dist/docs/references/reference-storage-postgresql.md +2 -0
- package/dist/docs/references/reference-storage-retention.md +180 -0
- package/dist/docs/references/reference-tools-vector-query-tool.md +2 -0
- package/dist/docs/references/reference-vectors-pg.md +2 -0
- package/dist/index.cjs +749 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +750 -3
- package/dist/index.js.map +1 -1
- package/dist/shared/config.d.ts +8 -1
- package/dist/shared/config.d.ts.map +1 -1
- package/dist/storage/db/index.d.ts +51 -0
- package/dist/storage/db/index.d.ts.map +1 -1
- package/dist/storage/domains/background-tasks/index.d.ts +19 -1
- package/dist/storage/domains/background-tasks/index.d.ts.map +1 -1
- package/dist/storage/domains/experiments/index.d.ts +31 -1
- package/dist/storage/domains/experiments/index.d.ts.map +1 -1
- package/dist/storage/domains/memory/index.d.ts +39 -1
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/notifications/index.d.ts +18 -1
- package/dist/storage/domains/notifications/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/index.d.ts +22 -1
- package/dist/storage/domains/observability/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +17 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/retention.d.ts +64 -0
- package/dist/storage/domains/observability/v-next/retention.d.ts.map +1 -0
- package/dist/storage/domains/schedules/index.d.ts +23 -1
- package/dist/storage/domains/schedules/index.d.ts.map +1 -1
- package/dist/storage/domains/scores/index.d.ts +18 -1
- package/dist/storage/domains/scores/index.d.ts.map +1 -1
- package/dist/storage/domains/workflows/index.d.ts +19 -1
- package/dist/storage/domains/workflows/index.d.ts.map +1 -1
- package/dist/storage/retention.d.ts +76 -0
- package/dist/storage/retention.d.ts.map +1 -0
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
2
|
-
import { BRANCH_SPAN_TYPES, createVectorErrorId, AgentsStorage, TABLE_AGENTS, TABLE_AGENT_VERSIONS, TABLE_SCHEMAS, createStorageErrorId, normalizePerPage, calculatePagination, BackgroundTasksStorage, TABLE_BACKGROUND_TASKS, BlobStore, TABLE_SKILL_BLOBS, ChannelsStorage, TABLE_CHANNEL_INSTALLATIONS, TABLE_CHANNEL_CONFIG, DatasetsStorage, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, DATASETS_SCHEMA, TABLE_CONFIGS, DATASET_ITEMS_SCHEMA, DATASET_VERSIONS_SCHEMA, ensureDate, safelyParseJSON, ExperimentsStorage, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, EXPERIMENTS_SCHEMA, EXPERIMENT_RESULTS_SCHEMA, FavoritesStorage, TABLE_FAVORITES, MCPClientsStorage, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, MCPServersStorage, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, MemoryStorage, TABLE_THREADS,
|
|
2
|
+
import { BRANCH_SPAN_TYPES, createVectorErrorId, AgentsStorage, TABLE_AGENTS, TABLE_AGENT_VERSIONS, TABLE_SCHEMAS, createStorageErrorId, normalizePerPage, calculatePagination, BackgroundTasksStorage, TABLE_BACKGROUND_TASKS, BlobStore, TABLE_SKILL_BLOBS, ChannelsStorage, TABLE_CHANNEL_INSTALLATIONS, TABLE_CHANNEL_CONFIG, DatasetsStorage, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, DATASETS_SCHEMA, TABLE_CONFIGS, DATASET_ITEMS_SCHEMA, DATASET_VERSIONS_SCHEMA, ensureDate, safelyParseJSON, ExperimentsStorage, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, EXPERIMENTS_SCHEMA, EXPERIMENT_RESULTS_SCHEMA, FavoritesStorage, TABLE_FAVORITES, MCPClientsStorage, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, MCPServersStorage, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, MemoryStorage, TABLE_THREADS, TABLE_RESOURCES, TABLE_MESSAGES, NotificationsStorage, TABLE_NOTIFICATIONS, ObservabilityStorage, TABLE_SPANS, listTracesArgsSchema, toTraceSpans, PromptBlocksStorage, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, SchedulesStorage, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, ScorerDefinitionsStorage, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, ScoresStorage, TABLE_SCORERS, SkillsStorage, TABLE_SKILLS, TABLE_SKILL_VERSIONS, ToolProviderConnectionsStorage, TABLE_TOOL_PROVIDER_CONNECTIONS, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, mergeWorkflowStepResult, WorkspacesStorage, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, MastraCompositeStore, TraceStatus, getDefaultValue, parseDuration, listBranchesArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listFeedbackArgsSchema, transformScoreRow as transformScoreRow$1, getSqlType, EntityType, METRIC_DISTINCT_COLUMNS } from '@mastra/core/storage';
|
|
3
3
|
import { parseSqlIdentifier, parseFieldKey } from '@mastra/core/utils';
|
|
4
4
|
import { MastraVector, validateTopK, validateUpsertInput } from '@mastra/core/vector';
|
|
5
5
|
import { Mutex } from 'async-mutex';
|
|
@@ -2517,6 +2517,11 @@ CREATE TRIGGER ${triggerName}
|
|
|
2517
2517
|
EXECUTE FUNCTION ${functionName}();`;
|
|
2518
2518
|
}
|
|
2519
2519
|
var schemaSetupRegistry = /* @__PURE__ */ new Map();
|
|
2520
|
+
function assertPositiveLimit(limit) {
|
|
2521
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
|
2522
|
+
throw new Error(`prune limit must be a positive integer; received ${limit}`);
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2520
2525
|
var PgDB = class extends MastraBase {
|
|
2521
2526
|
client;
|
|
2522
2527
|
schemaName;
|
|
@@ -3553,6 +3558,96 @@ Note: This migration may take some time for large tables.
|
|
|
3553
3558
|
async deleteData({ tableName }) {
|
|
3554
3559
|
return this.clearTable({ tableName });
|
|
3555
3560
|
}
|
|
3561
|
+
// ---------------------------------------------------------------------------
|
|
3562
|
+
// Retention helpers (prune)
|
|
3563
|
+
// ---------------------------------------------------------------------------
|
|
3564
|
+
/**
|
|
3565
|
+
* Deletes up to `limit` rows from `tableName` whose `column` value is strictly
|
|
3566
|
+
* older than `cutoff`, in a single bounded statement. Returns the number of
|
|
3567
|
+
* rows deleted so the caller's batch loop can decide whether the table is
|
|
3568
|
+
* drained.
|
|
3569
|
+
*
|
|
3570
|
+
* PostgreSQL has no `DELETE ... LIMIT`, so this targets a bounded set of
|
|
3571
|
+
* physical rows via the `ctid` system column (PG's row identity, analogous to
|
|
3572
|
+
* SQLite's `rowid`). `cutoff` is bound as a parameter — a `Date`/ISO-8601
|
|
3573
|
+
* string compared against a `timestamptz` anchor column, or a `number`
|
|
3574
|
+
* compared against a `bigint` epoch-ms anchor column.
|
|
3575
|
+
*/
|
|
3576
|
+
async pruneBatch({
|
|
3577
|
+
tableName,
|
|
3578
|
+
column,
|
|
3579
|
+
cutoff,
|
|
3580
|
+
limit
|
|
3581
|
+
}) {
|
|
3582
|
+
assertPositiveLimit(limit);
|
|
3583
|
+
const fullTableName = getTableName({ indexName: tableName, schemaName: getSchemaName(this.schemaName) });
|
|
3584
|
+
const parsedColumn = `"${parseSqlIdentifier(column, "column name")}"`;
|
|
3585
|
+
const sql = `
|
|
3586
|
+
DELETE FROM ${fullTableName}
|
|
3587
|
+
WHERE ctid IN (
|
|
3588
|
+
SELECT ctid FROM ${fullTableName}
|
|
3589
|
+
WHERE ${parsedColumn} < $1
|
|
3590
|
+
LIMIT $2
|
|
3591
|
+
)
|
|
3592
|
+
`;
|
|
3593
|
+
const result = await this.client.query(sql, [cutoff, limit]);
|
|
3594
|
+
return result.rowCount ?? 0;
|
|
3595
|
+
}
|
|
3596
|
+
/**
|
|
3597
|
+
* Deletes up to `limit` aged parent rows *and* their child rows together, in
|
|
3598
|
+
* a single transaction (used by whole-unit pruning such as experiments →
|
|
3599
|
+
* experiment_results). The aged parent IDs are selected first and both
|
|
3600
|
+
* deletes target that exact ID set, so a bound or abort between batches never
|
|
3601
|
+
* leaves a parent hollow (kept, but with its children gone) or children
|
|
3602
|
+
* orphaned.
|
|
3603
|
+
*/
|
|
3604
|
+
async pruneUnitsBatch({
|
|
3605
|
+
parentTable,
|
|
3606
|
+
parentKey,
|
|
3607
|
+
parentColumn,
|
|
3608
|
+
childTable,
|
|
3609
|
+
childForeignKey,
|
|
3610
|
+
cutoff,
|
|
3611
|
+
limit
|
|
3612
|
+
}) {
|
|
3613
|
+
assertPositiveLimit(limit);
|
|
3614
|
+
const schemaName = getSchemaName(this.schemaName);
|
|
3615
|
+
const fullChildTable = getTableName({ indexName: childTable, schemaName });
|
|
3616
|
+
const fullParentTable = getTableName({ indexName: parentTable, schemaName });
|
|
3617
|
+
const childFk = `"${parseSqlIdentifier(childForeignKey, "column name")}"`;
|
|
3618
|
+
const parentPk = `"${parseSqlIdentifier(parentKey, "column name")}"`;
|
|
3619
|
+
const parentCol = `"${parseSqlIdentifier(parentColumn, "column name")}"`;
|
|
3620
|
+
return this.client.tx(async (t) => {
|
|
3621
|
+
const rows = await t.manyOrNone(
|
|
3622
|
+
`SELECT ${parentPk} AS id FROM ${fullParentTable} WHERE ${parentCol} < $1 LIMIT $2 FOR UPDATE SKIP LOCKED`,
|
|
3623
|
+
[cutoff, limit]
|
|
3624
|
+
);
|
|
3625
|
+
const ids = rows.map((r) => r.id);
|
|
3626
|
+
if (ids.length === 0) return { parents: 0, children: 0 };
|
|
3627
|
+
const childResult = await t.query(`DELETE FROM ${fullChildTable} WHERE ${childFk} = ANY($1)`, [ids]);
|
|
3628
|
+
const parentResult = await t.query(`DELETE FROM ${fullParentTable} WHERE ${parentPk} = ANY($1)`, [ids]);
|
|
3629
|
+
return {
|
|
3630
|
+
parents: parentResult.rowCount ?? 0,
|
|
3631
|
+
children: childResult.rowCount ?? 0
|
|
3632
|
+
};
|
|
3633
|
+
});
|
|
3634
|
+
}
|
|
3635
|
+
/**
|
|
3636
|
+
* Creates a btree index on `column` for `tableName` if it does not already
|
|
3637
|
+
* exist, so age-based prune deletes stay fast. Delegates to {@link createIndex}
|
|
3638
|
+
* (which is a no-op when the index is present).
|
|
3639
|
+
*
|
|
3640
|
+
* The name is lowercased and truncated to Postgres' 63-byte identifier limit
|
|
3641
|
+
* (schema-prefixed names can exceed it), mirroring {@link buildConstraintName}.
|
|
3642
|
+
*/
|
|
3643
|
+
async ensureIndex({
|
|
3644
|
+
indexName,
|
|
3645
|
+
tableName,
|
|
3646
|
+
column
|
|
3647
|
+
}) {
|
|
3648
|
+
const name = buildConstraintName({ baseName: indexName });
|
|
3649
|
+
await this.createIndex({ name, table: tableName, columns: [column] });
|
|
3650
|
+
}
|
|
3556
3651
|
async executeUpdate(client, {
|
|
3557
3652
|
tableName,
|
|
3558
3653
|
keys,
|
|
@@ -4511,6 +4606,98 @@ var AgentsPG = class _AgentsPG extends AgentsStorage {
|
|
|
4511
4606
|
};
|
|
4512
4607
|
}
|
|
4513
4608
|
};
|
|
4609
|
+
var DEFAULT_BATCH_SIZE = 1e3;
|
|
4610
|
+
async function sleep(ms, signal) {
|
|
4611
|
+
if (ms <= 0 || signal?.aborted) return;
|
|
4612
|
+
await new Promise((resolve) => {
|
|
4613
|
+
const timer = setTimeout(() => {
|
|
4614
|
+
signal?.removeEventListener("abort", onAbort);
|
|
4615
|
+
resolve();
|
|
4616
|
+
}, ms);
|
|
4617
|
+
const onAbort = () => {
|
|
4618
|
+
clearTimeout(timer);
|
|
4619
|
+
resolve();
|
|
4620
|
+
};
|
|
4621
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
4622
|
+
});
|
|
4623
|
+
}
|
|
4624
|
+
function cutoffFor(policy, anchorType, now = Date.now()) {
|
|
4625
|
+
const cutoffMs = now - parseDuration(policy.maxAge);
|
|
4626
|
+
return anchorType === "epoch-ms" ? cutoffMs : new Date(cutoffMs);
|
|
4627
|
+
}
|
|
4628
|
+
async function runBatchedDelete({
|
|
4629
|
+
deleteBatch,
|
|
4630
|
+
batchSize,
|
|
4631
|
+
options
|
|
4632
|
+
}) {
|
|
4633
|
+
if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {
|
|
4634
|
+
throw new Error(`retention batchSize must be a positive integer; received ${batchSize}`);
|
|
4635
|
+
}
|
|
4636
|
+
let deleted = 0;
|
|
4637
|
+
let batches = 0;
|
|
4638
|
+
while (true) {
|
|
4639
|
+
if (options?.signal?.aborted) return { deleted, done: false };
|
|
4640
|
+
if (options?.maxBatches !== void 0 && batches >= options.maxBatches) return { deleted, done: false };
|
|
4641
|
+
let limit = batchSize;
|
|
4642
|
+
if (options?.maxRows !== void 0) {
|
|
4643
|
+
const remaining = options.maxRows - deleted;
|
|
4644
|
+
if (remaining <= 0) return { deleted, done: false };
|
|
4645
|
+
limit = Math.min(limit, remaining);
|
|
4646
|
+
}
|
|
4647
|
+
const affected = await deleteBatch(limit);
|
|
4648
|
+
deleted += affected;
|
|
4649
|
+
batches += 1;
|
|
4650
|
+
if (affected < limit) return { deleted, done: true };
|
|
4651
|
+
if (options?.pauseMs) {
|
|
4652
|
+
await sleep(options.pauseMs, options.signal);
|
|
4653
|
+
}
|
|
4654
|
+
}
|
|
4655
|
+
}
|
|
4656
|
+
async function runPrune({
|
|
4657
|
+
db,
|
|
4658
|
+
domain,
|
|
4659
|
+
targets,
|
|
4660
|
+
options
|
|
4661
|
+
}) {
|
|
4662
|
+
const results = [];
|
|
4663
|
+
const now = Date.now();
|
|
4664
|
+
for (const target of targets) {
|
|
4665
|
+
if (options?.signal?.aborted) {
|
|
4666
|
+
results.push({ domain, table: target.table, deleted: 0, done: false });
|
|
4667
|
+
continue;
|
|
4668
|
+
}
|
|
4669
|
+
const cutoff = cutoffFor(target.policy, target.anchorType, now);
|
|
4670
|
+
const batchSize = target.policy.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
4671
|
+
const { deleted, done } = await runBatchedDelete({
|
|
4672
|
+
deleteBatch: (limit) => db.pruneBatch({ tableName: target.table, column: target.column, cutoff, limit }),
|
|
4673
|
+
batchSize,
|
|
4674
|
+
options
|
|
4675
|
+
});
|
|
4676
|
+
results.push({ domain, table: target.table, deleted, done });
|
|
4677
|
+
}
|
|
4678
|
+
return results;
|
|
4679
|
+
}
|
|
4680
|
+
function resolveTargets({
|
|
4681
|
+
policies,
|
|
4682
|
+
descriptor,
|
|
4683
|
+
order
|
|
4684
|
+
}) {
|
|
4685
|
+
const targets = [];
|
|
4686
|
+
for (const key of order) {
|
|
4687
|
+
const policy = policies[key];
|
|
4688
|
+
const entry = descriptor[key];
|
|
4689
|
+
if (!policy || !entry) continue;
|
|
4690
|
+
targets.push({
|
|
4691
|
+
table: entry.table,
|
|
4692
|
+
column: entry.column,
|
|
4693
|
+
anchorType: entry.anchorType ?? "timestamp",
|
|
4694
|
+
policy
|
|
4695
|
+
});
|
|
4696
|
+
}
|
|
4697
|
+
return targets;
|
|
4698
|
+
}
|
|
4699
|
+
|
|
4700
|
+
// src/storage/domains/background-tasks/index.ts
|
|
4514
4701
|
function getSchemaName3(schema) {
|
|
4515
4702
|
return schema ? `"${schema}"` : '"public"';
|
|
4516
4703
|
}
|
|
@@ -4565,6 +4752,14 @@ var BackgroundTasksPG = class _BackgroundTasksPG extends BackgroundTasksStorage
|
|
|
4565
4752
|
#skipDefaultIndexes;
|
|
4566
4753
|
#indexes;
|
|
4567
4754
|
static MANAGED_TABLES = [TABLE_BACKGROUND_TASKS];
|
|
4755
|
+
/**
|
|
4756
|
+
* Completed background tasks accumulate as dead weight. Anchored on the
|
|
4757
|
+
* timezone-aware `completedAtZ` mirror column: NULL (in-flight) rows are
|
|
4758
|
+
* excluded automatically since `completedAtZ < cutoff` is false for NULL.
|
|
4759
|
+
*/
|
|
4760
|
+
static retentionTables = {
|
|
4761
|
+
backgroundTasks: { table: TABLE_BACKGROUND_TASKS, column: "completedAtZ", indexed: true }
|
|
4762
|
+
};
|
|
4568
4763
|
constructor(config) {
|
|
4569
4764
|
super();
|
|
4570
4765
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -4586,6 +4781,40 @@ var BackgroundTasksPG = class _BackgroundTasksPG extends BackgroundTasksStorage
|
|
|
4586
4781
|
await this.createDefaultIndexes();
|
|
4587
4782
|
await this.createCustomIndexes();
|
|
4588
4783
|
}
|
|
4784
|
+
/**
|
|
4785
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
4786
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
4787
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
4788
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
4789
|
+
* logged and pruning proceeds (correct, just slower).
|
|
4790
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
4791
|
+
* so its supporting index is not part of the default index set.
|
|
4792
|
+
*/
|
|
4793
|
+
async ensureRetentionIndexes(policies) {
|
|
4794
|
+
const prefix = this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
4795
|
+
for (const [key, entry] of Object.entries(_BackgroundTasksPG.retentionTables)) {
|
|
4796
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
4797
|
+
try {
|
|
4798
|
+
await this.#db.ensureIndex({
|
|
4799
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
4800
|
+
tableName: entry.table,
|
|
4801
|
+
column: entry.column
|
|
4802
|
+
});
|
|
4803
|
+
} catch (error) {
|
|
4804
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
4805
|
+
}
|
|
4806
|
+
}
|
|
4807
|
+
}
|
|
4808
|
+
/** Delete completed tasks older than the `backgroundTasks` policy's `maxAge`, batched. */
|
|
4809
|
+
async prune(policies, options) {
|
|
4810
|
+
await this.ensureRetentionIndexes(policies);
|
|
4811
|
+
const targets = resolveTargets({
|
|
4812
|
+
policies,
|
|
4813
|
+
descriptor: _BackgroundTasksPG.retentionTables,
|
|
4814
|
+
order: ["backgroundTasks"]
|
|
4815
|
+
});
|
|
4816
|
+
return runPrune({ db: this.#db, domain: "backgroundTasks", targets, options });
|
|
4817
|
+
}
|
|
4589
4818
|
static getDefaultIndexDefs(schemaPrefix) {
|
|
4590
4819
|
return [
|
|
4591
4820
|
{
|
|
@@ -6223,12 +6452,24 @@ var DatasetsPG = class _DatasetsPG extends DatasetsStorage {
|
|
|
6223
6452
|
await this.#db.clearTable({ tableName: TABLE_DATASETS });
|
|
6224
6453
|
}
|
|
6225
6454
|
};
|
|
6455
|
+
var DEFAULT_PRUNE_BATCH_SIZE = 1e3;
|
|
6226
6456
|
var ExperimentsPG = class _ExperimentsPG extends ExperimentsStorage {
|
|
6227
6457
|
#db;
|
|
6228
6458
|
#schema;
|
|
6229
6459
|
#skipDefaultIndexes;
|
|
6230
6460
|
#indexes;
|
|
6231
6461
|
static MANAGED_TABLES = [TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS];
|
|
6462
|
+
/**
|
|
6463
|
+
* Experiments prune as whole units: an aged experiment and its result rows go
|
|
6464
|
+
* together, mirroring `deleteExperiment`. Anchored on `completedAt` (not the
|
|
6465
|
+
* `completedAtZ` mirror, which carries a `DEFAULT NOW()` this domain never
|
|
6466
|
+
* overrides — it holds insert time even for running rows). `completedAt` is
|
|
6467
|
+
* written as a UTC ISO string and stays NULL while running, so
|
|
6468
|
+
* `completedAt < cutoff` is false for in-flight experiments.
|
|
6469
|
+
*/
|
|
6470
|
+
static retentionTables = {
|
|
6471
|
+
experiments: { table: TABLE_EXPERIMENTS, column: "completedAt", indexed: true }
|
|
6472
|
+
};
|
|
6232
6473
|
constructor(config) {
|
|
6233
6474
|
super();
|
|
6234
6475
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -6267,6 +6508,75 @@ var ExperimentsPG = class _ExperimentsPG extends ExperimentsStorage {
|
|
|
6267
6508
|
await this.createDefaultIndexes();
|
|
6268
6509
|
await this.createCustomIndexes();
|
|
6269
6510
|
}
|
|
6511
|
+
/**
|
|
6512
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
6513
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
6514
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
6515
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
6516
|
+
* logged and pruning proceeds (correct, just slower).
|
|
6517
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
6518
|
+
* so its supporting index is not part of the default index set.
|
|
6519
|
+
*/
|
|
6520
|
+
async ensureRetentionIndexes(policies) {
|
|
6521
|
+
const prefix = this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
6522
|
+
for (const [key, entry] of Object.entries(_ExperimentsPG.retentionTables)) {
|
|
6523
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
6524
|
+
try {
|
|
6525
|
+
await this.#db.ensureIndex({
|
|
6526
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
6527
|
+
tableName: entry.table,
|
|
6528
|
+
column: entry.column
|
|
6529
|
+
});
|
|
6530
|
+
} catch (error) {
|
|
6531
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
6532
|
+
}
|
|
6533
|
+
}
|
|
6534
|
+
}
|
|
6535
|
+
/**
|
|
6536
|
+
* Delete experiments whose `completedAt` is older than the policy's `maxAge`.
|
|
6537
|
+
*
|
|
6538
|
+
* Each batch selects up to `batchSize` aged experiments and deletes their
|
|
6539
|
+
* `experiment_results` rows and the experiment rows in one transaction —
|
|
6540
|
+
* mirroring `deleteExperiment` — so hitting `maxBatches`/`maxRows` or the
|
|
6541
|
+
* abort signal between batches never leaves a run hollow (parent kept,
|
|
6542
|
+
* results gone). NULL `completedAt` (still running) is excluded by the
|
|
6543
|
+
* `< cutoff` predicate. Bounds count whole experiments, not rows.
|
|
6544
|
+
*/
|
|
6545
|
+
async prune(policies, options) {
|
|
6546
|
+
const policy = policies["experiments"];
|
|
6547
|
+
if (!policy || options?.signal?.aborted) {
|
|
6548
|
+
return policy ? [
|
|
6549
|
+
{ domain: "experiments", table: TABLE_EXPERIMENT_RESULTS, deleted: 0, done: false },
|
|
6550
|
+
{ domain: "experiments", table: TABLE_EXPERIMENTS, deleted: 0, done: false }
|
|
6551
|
+
] : [];
|
|
6552
|
+
}
|
|
6553
|
+
await this.ensureRetentionIndexes(policies);
|
|
6554
|
+
const rawCutoff = cutoffFor(policy, "timestamp");
|
|
6555
|
+
const cutoff = rawCutoff instanceof Date ? rawCutoff.toISOString() : rawCutoff;
|
|
6556
|
+
const batchSize = policy.batchSize ?? DEFAULT_PRUNE_BATCH_SIZE;
|
|
6557
|
+
let childDeleted = 0;
|
|
6558
|
+
const parent = await runBatchedDelete({
|
|
6559
|
+
deleteBatch: async (limit) => {
|
|
6560
|
+
const { parents, children } = await this.#db.pruneUnitsBatch({
|
|
6561
|
+
parentTable: TABLE_EXPERIMENTS,
|
|
6562
|
+
parentKey: "id",
|
|
6563
|
+
parentColumn: "completedAt",
|
|
6564
|
+
childTable: TABLE_EXPERIMENT_RESULTS,
|
|
6565
|
+
childForeignKey: "experimentId",
|
|
6566
|
+
cutoff,
|
|
6567
|
+
limit
|
|
6568
|
+
});
|
|
6569
|
+
childDeleted += children;
|
|
6570
|
+
return parents;
|
|
6571
|
+
},
|
|
6572
|
+
batchSize,
|
|
6573
|
+
options
|
|
6574
|
+
});
|
|
6575
|
+
return [
|
|
6576
|
+
{ domain: "experiments", table: TABLE_EXPERIMENT_RESULTS, deleted: childDeleted, done: parent.done },
|
|
6577
|
+
{ domain: "experiments", table: TABLE_EXPERIMENTS, deleted: parent.deleted, done: parent.done }
|
|
6578
|
+
];
|
|
6579
|
+
}
|
|
6270
6580
|
getDefaultIndexDefinitions() {
|
|
6271
6581
|
return [
|
|
6272
6582
|
{ name: "idx_experiments_datasetid", table: TABLE_EXPERIMENTS, columns: ["datasetId"] },
|
|
@@ -8435,6 +8745,18 @@ function dedupeMessagesForSave(messages) {
|
|
|
8435
8745
|
}
|
|
8436
8746
|
var MemoryPG = class _MemoryPG extends MemoryStorage {
|
|
8437
8747
|
supportsObservationalMemory = true;
|
|
8748
|
+
/**
|
|
8749
|
+
* Retention-eligible tables. `threads`, `messages`, and `resources` all anchor
|
|
8750
|
+
* on the timezone-aware `createdAtZ` mirror column (kept in sync by triggers),
|
|
8751
|
+
* and are indexed for fast batched deletes. Cascade order is enforced in
|
|
8752
|
+
* `prune()` (children before threads), not here. Observational memory has no
|
|
8753
|
+
* timestamp anchor and is deliberately excluded.
|
|
8754
|
+
*/
|
|
8755
|
+
static retentionTables = {
|
|
8756
|
+
messages: { table: TABLE_MESSAGES, column: "createdAtZ", indexed: true },
|
|
8757
|
+
resources: { table: TABLE_RESOURCES, column: "createdAtZ", indexed: true },
|
|
8758
|
+
threads: { table: TABLE_THREADS, column: "createdAtZ", indexed: true }
|
|
8759
|
+
};
|
|
8438
8760
|
#db;
|
|
8439
8761
|
#schema;
|
|
8440
8762
|
#skipDefaultIndexes;
|
|
@@ -8480,6 +8802,30 @@ var MemoryPG = class _MemoryPG extends MemoryStorage {
|
|
|
8480
8802
|
await this.createDefaultIndexes();
|
|
8481
8803
|
await this.createCustomIndexes();
|
|
8482
8804
|
}
|
|
8805
|
+
/**
|
|
8806
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
8807
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
8808
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
8809
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
8810
|
+
* logged and pruning proceeds (correct, just slower).
|
|
8811
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
8812
|
+
* so its supporting index is not part of the default index set.
|
|
8813
|
+
*/
|
|
8814
|
+
async ensureRetentionIndexes(policies) {
|
|
8815
|
+
const prefix = this.#schema && this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
8816
|
+
for (const [key, entry] of Object.entries(_MemoryPG.retentionTables)) {
|
|
8817
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
8818
|
+
try {
|
|
8819
|
+
await this.#db.ensureIndex({
|
|
8820
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
8821
|
+
tableName: entry.table,
|
|
8822
|
+
column: entry.column
|
|
8823
|
+
});
|
|
8824
|
+
} catch (error) {
|
|
8825
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
8826
|
+
}
|
|
8827
|
+
}
|
|
8828
|
+
}
|
|
8483
8829
|
/**
|
|
8484
8830
|
* Returns default index definitions for the memory domain tables.
|
|
8485
8831
|
* @param schemaPrefix - Prefix for index names (e.g. "my_schema_" or "")
|
|
@@ -8580,6 +8926,76 @@ var MemoryPG = class _MemoryPG extends MemoryStorage {
|
|
|
8580
8926
|
await this.#db.clearTable({ tableName: TABLE_THREADS });
|
|
8581
8927
|
await this.#db.clearTable({ tableName: TABLE_RESOURCES });
|
|
8582
8928
|
}
|
|
8929
|
+
/**
|
|
8930
|
+
* Deletes rows older than the configured `maxAge` per table, in bounded,
|
|
8931
|
+
* batched, cancellable chunks. Tables are pruned children-first (messages and
|
|
8932
|
+
* resources before threads) since PostgreSQL has no FK cascade in this schema.
|
|
8933
|
+
* Unset tables are kept forever.
|
|
8934
|
+
*
|
|
8935
|
+
* When a `messages` policy is set, semantic-recall embeddings for pruned
|
|
8936
|
+
* messages are also swept from same-schema `memory_messages*` vector tables
|
|
8937
|
+
* (best-effort, mirroring `deleteThread`). Embeddings held in an external
|
|
8938
|
+
* vector store are out of reach and must be pruned by the operator.
|
|
8939
|
+
*/
|
|
8940
|
+
async prune(policies, options) {
|
|
8941
|
+
await this.ensureRetentionIndexes(policies);
|
|
8942
|
+
const targets = resolveTargets({
|
|
8943
|
+
policies,
|
|
8944
|
+
descriptor: _MemoryPG.retentionTables,
|
|
8945
|
+
order: ["messages", "resources", "threads"]
|
|
8946
|
+
});
|
|
8947
|
+
const results = await runPrune({ db: this.#db, domain: "memory", targets, options });
|
|
8948
|
+
if (policies["messages"]) {
|
|
8949
|
+
await this.pruneOrphanedVectorRows(policies["messages"], options);
|
|
8950
|
+
}
|
|
8951
|
+
return results;
|
|
8952
|
+
}
|
|
8953
|
+
/**
|
|
8954
|
+
* Best-effort sweep of semantic-recall vector rows whose source message no
|
|
8955
|
+
* longer exists (e.g. it was just pruned), so recall doesn't keep returning
|
|
8956
|
+
* embeddings that resolve to nothing. Only same-schema default vector tables
|
|
8957
|
+
* (`memory_messages*`) are covered — the same set `deleteThread` cleans up.
|
|
8958
|
+
* Failures are logged, never thrown: vector cleanup must not fail the prune.
|
|
8959
|
+
*/
|
|
8960
|
+
async pruneOrphanedVectorRows(policy, options) {
|
|
8961
|
+
try {
|
|
8962
|
+
const schemaName = this.#schema || "public";
|
|
8963
|
+
const vectorTables = await this.#db.client.manyOrNone(
|
|
8964
|
+
`
|
|
8965
|
+
SELECT tablename
|
|
8966
|
+
FROM pg_tables
|
|
8967
|
+
WHERE schemaname = $1
|
|
8968
|
+
AND (tablename = 'memory_messages' OR tablename LIKE 'memory_messages_%')
|
|
8969
|
+
`,
|
|
8970
|
+
[schemaName]
|
|
8971
|
+
);
|
|
8972
|
+
const messagesTable = getTableName4({ indexName: TABLE_MESSAGES, schemaName: getSchemaName4(this.#schema) });
|
|
8973
|
+
for (const { tablename } of vectorTables) {
|
|
8974
|
+
const vectorTableName = getTableName4({ indexName: tablename, schemaName: getSchemaName4(this.#schema) });
|
|
8975
|
+
await runBatchedDelete({
|
|
8976
|
+
deleteBatch: async (limit) => {
|
|
8977
|
+
const result = await this.#db.client.query(
|
|
8978
|
+
`
|
|
8979
|
+
DELETE FROM ${vectorTableName}
|
|
8980
|
+
WHERE ctid IN (
|
|
8981
|
+
SELECT v.ctid FROM ${vectorTableName} v
|
|
8982
|
+
WHERE v.metadata->>'message_id' IS NOT NULL
|
|
8983
|
+
AND NOT EXISTS (SELECT 1 FROM ${messagesTable} m WHERE m.id = v.metadata->>'message_id')
|
|
8984
|
+
LIMIT $1
|
|
8985
|
+
)
|
|
8986
|
+
`,
|
|
8987
|
+
[limit]
|
|
8988
|
+
);
|
|
8989
|
+
return result.rowCount ?? 0;
|
|
8990
|
+
},
|
|
8991
|
+
batchSize: policy.batchSize ?? 1e3,
|
|
8992
|
+
options
|
|
8993
|
+
});
|
|
8994
|
+
}
|
|
8995
|
+
} catch (error) {
|
|
8996
|
+
this.logger?.warn?.("Failed to sweep orphaned semantic-recall vector rows after prune:", error);
|
|
8997
|
+
}
|
|
8998
|
+
}
|
|
8583
8999
|
/**
|
|
8584
9000
|
* Normalizes message row from database by applying createdAtZ fallback
|
|
8585
9001
|
*/
|
|
@@ -10891,6 +11307,13 @@ var NotificationsPG = class _NotificationsPG extends NotificationsStorage {
|
|
|
10891
11307
|
#skipDefaultIndexes;
|
|
10892
11308
|
#indexes;
|
|
10893
11309
|
static MANAGED_TABLES = [TABLE_NOTIFICATIONS];
|
|
11310
|
+
/**
|
|
11311
|
+
* Notifications are an append-only event feed that grows unbounded. Single
|
|
11312
|
+
* table, anchored on the timezone-aware `createdAtZ` mirror column.
|
|
11313
|
+
*/
|
|
11314
|
+
static retentionTables = {
|
|
11315
|
+
notifications: { table: TABLE_NOTIFICATIONS, column: "createdAtZ", indexed: true }
|
|
11316
|
+
};
|
|
10894
11317
|
constructor(config) {
|
|
10895
11318
|
super();
|
|
10896
11319
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -10907,6 +11330,40 @@ var NotificationsPG = class _NotificationsPG extends NotificationsStorage {
|
|
|
10907
11330
|
await this.createDefaultIndexes();
|
|
10908
11331
|
await this.createCustomIndexes();
|
|
10909
11332
|
}
|
|
11333
|
+
/**
|
|
11334
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
11335
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
11336
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
11337
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
11338
|
+
* logged and pruning proceeds (correct, just slower).
|
|
11339
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
11340
|
+
* so its supporting index is not part of the default index set.
|
|
11341
|
+
*/
|
|
11342
|
+
async ensureRetentionIndexes(policies) {
|
|
11343
|
+
const prefix = this.#schema && this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
11344
|
+
for (const [key, entry] of Object.entries(_NotificationsPG.retentionTables)) {
|
|
11345
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
11346
|
+
try {
|
|
11347
|
+
await this.#db.ensureIndex({
|
|
11348
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
11349
|
+
tableName: entry.table,
|
|
11350
|
+
column: entry.column
|
|
11351
|
+
});
|
|
11352
|
+
} catch (error) {
|
|
11353
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
11354
|
+
}
|
|
11355
|
+
}
|
|
11356
|
+
}
|
|
11357
|
+
/** Delete notifications older than the `notifications` policy's `maxAge`, batched. */
|
|
11358
|
+
async prune(policies, options) {
|
|
11359
|
+
await this.ensureRetentionIndexes(policies);
|
|
11360
|
+
const targets = resolveTargets({
|
|
11361
|
+
policies,
|
|
11362
|
+
descriptor: _NotificationsPG.retentionTables,
|
|
11363
|
+
order: ["notifications"]
|
|
11364
|
+
});
|
|
11365
|
+
return runPrune({ db: this.#db, domain: "notifications", targets, options });
|
|
11366
|
+
}
|
|
10910
11367
|
static getDefaultIndexDefs(schemaPrefix) {
|
|
10911
11368
|
return [
|
|
10912
11369
|
{
|
|
@@ -11148,6 +11605,14 @@ var NotificationsPG = class _NotificationsPG extends NotificationsStorage {
|
|
|
11148
11605
|
}
|
|
11149
11606
|
};
|
|
11150
11607
|
var ObservabilityPG = class _ObservabilityPG extends ObservabilityStorage {
|
|
11608
|
+
/**
|
|
11609
|
+
* Retention-eligible tables. The observability domain has a single physical
|
|
11610
|
+
* table (`mastra_ai_spans`); spans anchor on the timezone-aware `startedAtZ`
|
|
11611
|
+
* mirror column and are indexed for fast batched deletes.
|
|
11612
|
+
*/
|
|
11613
|
+
static retentionTables = {
|
|
11614
|
+
spans: { table: TABLE_SPANS, column: "startedAtZ", indexed: true }
|
|
11615
|
+
};
|
|
11151
11616
|
#db;
|
|
11152
11617
|
#schema;
|
|
11153
11618
|
#skipDefaultIndexes;
|
|
@@ -11172,6 +11637,30 @@ var ObservabilityPG = class _ObservabilityPG extends ObservabilityStorage {
|
|
|
11172
11637
|
await this.createDefaultIndexes();
|
|
11173
11638
|
await this.createCustomIndexes();
|
|
11174
11639
|
}
|
|
11640
|
+
/**
|
|
11641
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
11642
|
+
* anchor column so age-based `prune()` deletes stay fast on large span tables.
|
|
11643
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
11644
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
11645
|
+
* logged and pruning proceeds (correct, just slower).
|
|
11646
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
11647
|
+
* so its supporting index is not part of the default index set.
|
|
11648
|
+
*/
|
|
11649
|
+
async ensureRetentionIndexes(policies) {
|
|
11650
|
+
const prefix = this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
11651
|
+
for (const [key, entry] of Object.entries(_ObservabilityPG.retentionTables)) {
|
|
11652
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
11653
|
+
try {
|
|
11654
|
+
await this.#db.ensureIndex({
|
|
11655
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
11656
|
+
tableName: entry.table,
|
|
11657
|
+
column: entry.column
|
|
11658
|
+
});
|
|
11659
|
+
} catch (error) {
|
|
11660
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
11661
|
+
}
|
|
11662
|
+
}
|
|
11663
|
+
}
|
|
11175
11664
|
/**
|
|
11176
11665
|
* Returns default index definitions for the observability domain tables.
|
|
11177
11666
|
* @param schemaPrefix - Prefix for index names (e.g. "my_schema_" or "")
|
|
@@ -11316,6 +11805,19 @@ var ObservabilityPG = class _ObservabilityPG extends ObservabilityStorage {
|
|
|
11316
11805
|
async dangerouslyClearAll() {
|
|
11317
11806
|
await this.#db.clearTable({ tableName: TABLE_SPANS });
|
|
11318
11807
|
}
|
|
11808
|
+
/**
|
|
11809
|
+
* Deletes spans older than the configured `maxAge`, in bounded, batched,
|
|
11810
|
+
* cancellable chunks. Unset = keep forever.
|
|
11811
|
+
*/
|
|
11812
|
+
async prune(policies, options) {
|
|
11813
|
+
await this.ensureRetentionIndexes(policies);
|
|
11814
|
+
const targets = resolveTargets({
|
|
11815
|
+
policies,
|
|
11816
|
+
descriptor: _ObservabilityPG.retentionTables,
|
|
11817
|
+
order: ["spans"]
|
|
11818
|
+
});
|
|
11819
|
+
return runPrune({ db: this.#db, domain: "observability", targets, options });
|
|
11820
|
+
}
|
|
11319
11821
|
get tracingStrategy() {
|
|
11320
11822
|
return {
|
|
11321
11823
|
preferred: "batch-with-updates",
|
|
@@ -14012,6 +14514,78 @@ async function setupPartitioning(client, schema, options = {}) {
|
|
|
14012
14514
|
return "native";
|
|
14013
14515
|
}
|
|
14014
14516
|
}
|
|
14517
|
+
function retentionCutoff(policy, now = Date.now()) {
|
|
14518
|
+
return new Date(now - parseDuration(policy.maxAge));
|
|
14519
|
+
}
|
|
14520
|
+
async function listChildPartitions(client, schema, table) {
|
|
14521
|
+
const rows = await client.manyOrNone(
|
|
14522
|
+
`SELECT c.relname AS "name", pg_get_expr(c.relpartbound, c.oid) AS "bound"
|
|
14523
|
+
FROM pg_inherits i
|
|
14524
|
+
JOIN pg_class c ON c.oid = i.inhrelid
|
|
14525
|
+
JOIN pg_class p ON p.oid = i.inhparent
|
|
14526
|
+
JOIN pg_namespace n ON n.oid = p.relnamespace
|
|
14527
|
+
WHERE n.nspname = $1 AND p.relname = $2`,
|
|
14528
|
+
[schema, table]
|
|
14529
|
+
);
|
|
14530
|
+
const partitions = [];
|
|
14531
|
+
for (const row of rows ?? []) {
|
|
14532
|
+
const match = /TO \('([^']+)'\)/.exec(row.bound ?? "");
|
|
14533
|
+
if (!match?.[1]) continue;
|
|
14534
|
+
const upperBound = new Date(match[1]);
|
|
14535
|
+
if (Number.isNaN(upperBound.getTime())) continue;
|
|
14536
|
+
partitions.push({ name: row.name, upperBound });
|
|
14537
|
+
}
|
|
14538
|
+
return partitions.sort((a, b) => a.upperBound.getTime() - b.upperBound.getTime());
|
|
14539
|
+
}
|
|
14540
|
+
async function prunePartitionedTable({
|
|
14541
|
+
client,
|
|
14542
|
+
schema,
|
|
14543
|
+
table,
|
|
14544
|
+
cutoff,
|
|
14545
|
+
options
|
|
14546
|
+
}) {
|
|
14547
|
+
const partitions = await listChildPartitions(client, schema, table);
|
|
14548
|
+
const droppable = partitions.filter((p) => p.upperBound.getTime() <= cutoff.getTime());
|
|
14549
|
+
let deleted = 0;
|
|
14550
|
+
let batches = 0;
|
|
14551
|
+
for (const partition of droppable) {
|
|
14552
|
+
if (options?.signal?.aborted) return { deleted, done: false };
|
|
14553
|
+
if (options?.maxBatches !== void 0 && batches >= options.maxBatches) return { deleted, done: false };
|
|
14554
|
+
if (options?.maxRows !== void 0 && deleted >= options.maxRows) return { deleted, done: false };
|
|
14555
|
+
const child = qualifiedName(schema, partition.name);
|
|
14556
|
+
const parent = qualifiedTable(schema, table);
|
|
14557
|
+
const row = await client.one(`SELECT count(*)::int AS "n" FROM ${child}`);
|
|
14558
|
+
await client.none(`ALTER TABLE ${parent} DETACH PARTITION ${child}`);
|
|
14559
|
+
await client.none(`DROP TABLE IF EXISTS ${child}`);
|
|
14560
|
+
deleted += row.n;
|
|
14561
|
+
batches += 1;
|
|
14562
|
+
}
|
|
14563
|
+
return { deleted, done: true };
|
|
14564
|
+
}
|
|
14565
|
+
async function pruneTimescaleTable({
|
|
14566
|
+
client,
|
|
14567
|
+
schema,
|
|
14568
|
+
table,
|
|
14569
|
+
cutoff,
|
|
14570
|
+
options
|
|
14571
|
+
}) {
|
|
14572
|
+
if (options?.signal?.aborted || options?.maxBatches === 0) return { deleted: 0, done: false };
|
|
14573
|
+
const tableExpr = qualifiedTable(schema, table);
|
|
14574
|
+
const chunks = await client.manyOrNone(
|
|
14575
|
+
`SELECT show_chunks($1::regclass, older_than => $2::timestamptz)::text AS "chunk"`,
|
|
14576
|
+
[tableExpr, cutoff.toISOString()]
|
|
14577
|
+
);
|
|
14578
|
+
let deleted = 0;
|
|
14579
|
+
for (const { chunk } of chunks ?? []) {
|
|
14580
|
+
const row = await client.one(`SELECT count(*)::int AS "n" FROM ${chunk}`);
|
|
14581
|
+
deleted += row.n;
|
|
14582
|
+
}
|
|
14583
|
+
await client.none(`SELECT drop_chunks($1::regclass, older_than => $2::timestamptz)`, [
|
|
14584
|
+
tableExpr,
|
|
14585
|
+
cutoff.toISOString()
|
|
14586
|
+
]);
|
|
14587
|
+
return { deleted, done: true };
|
|
14588
|
+
}
|
|
14015
14589
|
function applyScoreFilters(acc, filters) {
|
|
14016
14590
|
applyCommonFilters(acc, filters);
|
|
14017
14591
|
applySingleOrArrayFilter(acc, "scorerId", filters?.scorerId);
|
|
@@ -14758,7 +15332,7 @@ function wrapError(op, error, details) {
|
|
|
14758
15332
|
error
|
|
14759
15333
|
);
|
|
14760
15334
|
}
|
|
14761
|
-
var ObservabilityStoragePostgresVNext = class extends ObservabilityStorage {
|
|
15335
|
+
var ObservabilityStoragePostgresVNext = class _ObservabilityStoragePostgresVNext extends ObservabilityStorage {
|
|
14762
15336
|
#client;
|
|
14763
15337
|
#schema;
|
|
14764
15338
|
#partitioning;
|
|
@@ -14842,6 +15416,50 @@ var ObservabilityStoragePostgresVNext = class extends ObservabilityStorage {
|
|
|
14842
15416
|
get partitionMode() {
|
|
14843
15417
|
return this.#partitionMode;
|
|
14844
15418
|
}
|
|
15419
|
+
// -------------------------------------------------------------------------
|
|
15420
|
+
// Retention
|
|
15421
|
+
// -------------------------------------------------------------------------
|
|
15422
|
+
/**
|
|
15423
|
+
* All five signal tables are insert-only growth tables. The anchor column is
|
|
15424
|
+
* each table's partition / chunk key, so age-based expiry drops whole day
|
|
15425
|
+
* partitions instead of deleting rows (see `./retention.ts`). `indexed: true`
|
|
15426
|
+
* reflects that expiry never scans — it acts on partition bounds.
|
|
15427
|
+
*/
|
|
15428
|
+
static retentionTables = {
|
|
15429
|
+
spans: { table: TABLE_SPAN_EVENTS, column: "endedAt", indexed: true },
|
|
15430
|
+
metrics: { table: TABLE_METRIC_EVENTS, column: "timestamp", indexed: true },
|
|
15431
|
+
logs: { table: TABLE_LOG_EVENTS, column: "timestamp", indexed: true },
|
|
15432
|
+
scores: { table: TABLE_SCORE_EVENTS, column: "timestamp", indexed: true },
|
|
15433
|
+
feedback: { table: TABLE_FEEDBACK_EVENTS, column: "timestamp", indexed: true }
|
|
15434
|
+
};
|
|
15435
|
+
/**
|
|
15436
|
+
* Expire signal events older than each table's `maxAge` by dropping whole
|
|
15437
|
+
* day partitions (native / pg_partman) or chunks (Timescale). Only
|
|
15438
|
+
* partitions wholly older than the cutoff are dropped, so the effective
|
|
15439
|
+
* granularity is one day. Each partition drop counts as one batch for
|
|
15440
|
+
* `maxBatches` / `maxRows` / abort-signal purposes; `deleted` reports the
|
|
15441
|
+
* row count of the dropped partitions.
|
|
15442
|
+
*/
|
|
15443
|
+
async prune(policies, options) {
|
|
15444
|
+
return this.#run("VNEXT_PRUNE", async () => {
|
|
15445
|
+
const mode = this.#partitionMode ??= await resolveMode(this.#client, this.#partitioning);
|
|
15446
|
+
const results = [];
|
|
15447
|
+
const now = Date.now();
|
|
15448
|
+
for (const [key, entry] of Object.entries(_ObservabilityStoragePostgresVNext.retentionTables)) {
|
|
15449
|
+
const policy = policies[key];
|
|
15450
|
+
if (!policy) continue;
|
|
15451
|
+
if (options?.signal?.aborted) {
|
|
15452
|
+
results.push({ domain: "observability", table: entry.table, deleted: 0, done: false });
|
|
15453
|
+
continue;
|
|
15454
|
+
}
|
|
15455
|
+
const cutoff = retentionCutoff(policy, now);
|
|
15456
|
+
const args = { client: this.#client, schema: this.#schema, table: entry.table, cutoff, options };
|
|
15457
|
+
const outcome = mode === "timescale" ? await pruneTimescaleTable(args) : await prunePartitionedTable(args);
|
|
15458
|
+
results.push({ domain: "observability", table: entry.table, deleted: outcome.deleted, done: outcome.done });
|
|
15459
|
+
}
|
|
15460
|
+
return results;
|
|
15461
|
+
});
|
|
15462
|
+
}
|
|
14845
15463
|
get observabilityStrategy() {
|
|
14846
15464
|
return { preferred: "insert-only", supported: ["insert-only"] };
|
|
14847
15465
|
}
|
|
@@ -15786,6 +16404,15 @@ var SchedulesPG = class _SchedulesPG extends SchedulesStorage {
|
|
|
15786
16404
|
#indexes;
|
|
15787
16405
|
/** Tables managed by this domain */
|
|
15788
16406
|
static MANAGED_TABLES = [TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS];
|
|
16407
|
+
/**
|
|
16408
|
+
* The fire/run history (`schedule_triggers`, one row per fire) is the growth
|
|
16409
|
+
* table; schedule definitions are config and excluded. Anchored on
|
|
16410
|
+
* `actual_fire_at`, a bigint epoch-ms column (numeric comparison, not
|
|
16411
|
+
* timestamptz).
|
|
16412
|
+
*/
|
|
16413
|
+
static retentionTables = {
|
|
16414
|
+
triggers: { table: TABLE_SCHEDULE_TRIGGERS, column: "actual_fire_at", indexed: true, anchorType: "epoch-ms" }
|
|
16415
|
+
};
|
|
15789
16416
|
constructor(config) {
|
|
15790
16417
|
super();
|
|
15791
16418
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -15807,6 +16434,43 @@ var SchedulesPG = class _SchedulesPG extends SchedulesStorage {
|
|
|
15807
16434
|
await this.createDefaultIndexes();
|
|
15808
16435
|
await this.createCustomIndexes();
|
|
15809
16436
|
}
|
|
16437
|
+
/**
|
|
16438
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
16439
|
+
* anchor column so age-based `prune()` deletes stay fast. The default
|
|
16440
|
+
* composite index leads with `schedule_id`, so a bare `actual_fire_at` range
|
|
16441
|
+
* scan can't use it. Called from the prune path (not init) so only
|
|
16442
|
+
* deployments that configure retention pay the index's write/disk overhead.
|
|
16443
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
16444
|
+
* so its supporting index is not part of the default index set.
|
|
16445
|
+
*/
|
|
16446
|
+
async ensureRetentionIndexes(policies) {
|
|
16447
|
+
const prefix = this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
16448
|
+
for (const [key, entry] of Object.entries(_SchedulesPG.retentionTables)) {
|
|
16449
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
16450
|
+
try {
|
|
16451
|
+
await this.#db.ensureIndex({
|
|
16452
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
16453
|
+
tableName: entry.table,
|
|
16454
|
+
column: entry.column
|
|
16455
|
+
});
|
|
16456
|
+
} catch (error) {
|
|
16457
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
16458
|
+
}
|
|
16459
|
+
}
|
|
16460
|
+
}
|
|
16461
|
+
/**
|
|
16462
|
+
* Delete trigger (fire history) rows whose `actual_fire_at` is older than the
|
|
16463
|
+
* `triggers` policy's `maxAge`, batched. Schedule definitions are never pruned.
|
|
16464
|
+
*/
|
|
16465
|
+
async prune(policies, options) {
|
|
16466
|
+
await this.ensureRetentionIndexes(policies);
|
|
16467
|
+
const targets = resolveTargets({
|
|
16468
|
+
policies,
|
|
16469
|
+
descriptor: _SchedulesPG.retentionTables,
|
|
16470
|
+
order: ["triggers"]
|
|
16471
|
+
});
|
|
16472
|
+
return runPrune({ db: this.#db, domain: "schedules", targets, options });
|
|
16473
|
+
}
|
|
15810
16474
|
/**
|
|
15811
16475
|
* Returns default index definitions for the schedules domain.
|
|
15812
16476
|
* @param schemaPrefix - Prefix for index names (e.g. "my_schema_" or "")
|
|
@@ -16737,6 +17401,13 @@ var ScoresPG = class _ScoresPG extends ScoresStorage {
|
|
|
16737
17401
|
#indexes;
|
|
16738
17402
|
/** Tables managed by this domain */
|
|
16739
17403
|
static MANAGED_TABLES = [TABLE_SCORERS];
|
|
17404
|
+
/**
|
|
17405
|
+
* Scorer results accumulate as evals run. Single table, anchored on the
|
|
17406
|
+
* timezone-aware `createdAtZ` mirror column (kept in sync by triggers).
|
|
17407
|
+
*/
|
|
17408
|
+
static retentionTables = {
|
|
17409
|
+
scorers: { table: TABLE_SCORERS, column: "createdAtZ", indexed: true }
|
|
17410
|
+
};
|
|
16740
17411
|
constructor(config) {
|
|
16741
17412
|
super();
|
|
16742
17413
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -16755,6 +17426,30 @@ var ScoresPG = class _ScoresPG extends ScoresStorage {
|
|
|
16755
17426
|
await this.createDefaultIndexes();
|
|
16756
17427
|
await this.createCustomIndexes();
|
|
16757
17428
|
}
|
|
17429
|
+
/**
|
|
17430
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
17431
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
17432
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
17433
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
17434
|
+
* logged and pruning proceeds (correct, just slower).
|
|
17435
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
17436
|
+
* so its supporting index is not part of the default index set.
|
|
17437
|
+
*/
|
|
17438
|
+
async ensureRetentionIndexes(policies) {
|
|
17439
|
+
const prefix = this.#schema && this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
17440
|
+
for (const [key, entry] of Object.entries(_ScoresPG.retentionTables)) {
|
|
17441
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
17442
|
+
try {
|
|
17443
|
+
await this.#db.ensureIndex({
|
|
17444
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
17445
|
+
tableName: entry.table,
|
|
17446
|
+
column: entry.column
|
|
17447
|
+
});
|
|
17448
|
+
} catch (error) {
|
|
17449
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
17450
|
+
}
|
|
17451
|
+
}
|
|
17452
|
+
}
|
|
16758
17453
|
/**
|
|
16759
17454
|
* Returns default index definitions for the scores domain tables.
|
|
16760
17455
|
* @param schemaPrefix - Prefix for index names (e.g. "my_schema_" or "")
|
|
@@ -16829,6 +17524,16 @@ var ScoresPG = class _ScoresPG extends ScoresStorage {
|
|
|
16829
17524
|
async dangerouslyClearAll() {
|
|
16830
17525
|
await this.#db.clearTable({ tableName: TABLE_SCORERS });
|
|
16831
17526
|
}
|
|
17527
|
+
/** Delete scorer results older than the `scorers` policy's `maxAge`, batched. */
|
|
17528
|
+
async prune(policies, options) {
|
|
17529
|
+
await this.ensureRetentionIndexes(policies);
|
|
17530
|
+
const targets = resolveTargets({
|
|
17531
|
+
policies,
|
|
17532
|
+
descriptor: _ScoresPG.retentionTables,
|
|
17533
|
+
order: ["scorers"]
|
|
17534
|
+
});
|
|
17535
|
+
return runPrune({ db: this.#db, domain: "scores", targets, options });
|
|
17536
|
+
}
|
|
16832
17537
|
async getScoreById({ id }) {
|
|
16833
17538
|
try {
|
|
16834
17539
|
const result = await this.#db.client.oneOrNone(
|
|
@@ -18150,6 +18855,14 @@ var WorkflowsPG = class _WorkflowsPG extends WorkflowsStorage {
|
|
|
18150
18855
|
#indexes;
|
|
18151
18856
|
/** Tables managed by this domain */
|
|
18152
18857
|
static MANAGED_TABLES = [TABLE_WORKFLOW_SNAPSHOT];
|
|
18858
|
+
/**
|
|
18859
|
+
* Workflow run snapshots accumulate as runs execute. Anchored on the
|
|
18860
|
+
* timezone-aware `updatedAtZ` mirror column (last activity) so suspended or
|
|
18861
|
+
* long-running runs are not pruned by start age.
|
|
18862
|
+
*/
|
|
18863
|
+
static retentionTables = {
|
|
18864
|
+
workflowSnapshot: { table: TABLE_WORKFLOW_SNAPSHOT, column: "updatedAtZ", indexed: true }
|
|
18865
|
+
};
|
|
18153
18866
|
constructor(config) {
|
|
18154
18867
|
super();
|
|
18155
18868
|
const { client, schemaName, skipDefaultIndexes, indexes } = resolvePgConfig(config);
|
|
@@ -18221,6 +18934,40 @@ var WorkflowsPG = class _WorkflowsPG extends WorkflowsStorage {
|
|
|
18221
18934
|
await this.createDefaultIndexes();
|
|
18222
18935
|
await this.createCustomIndexes();
|
|
18223
18936
|
}
|
|
18937
|
+
/**
|
|
18938
|
+
* Lazily ensures a btree index exists on each configured policy's retention
|
|
18939
|
+
* anchor column so age-based `prune()` deletes stay fast on large tables.
|
|
18940
|
+
* Called from the prune path (not init) so only deployments that configure
|
|
18941
|
+
* retention pay the index's write/disk overhead. Best-effort: failures are
|
|
18942
|
+
* logged and pruning proceeds (correct, just slower).
|
|
18943
|
+
* Created even with `skipDefaultIndexes` — retention is an explicit opt-in,
|
|
18944
|
+
* so its supporting index is not part of the default index set.
|
|
18945
|
+
*/
|
|
18946
|
+
async ensureRetentionIndexes(policies) {
|
|
18947
|
+
const prefix = this.#schema && this.#schema !== "public" ? `${this.#schema}_` : "";
|
|
18948
|
+
for (const [key, entry] of Object.entries(_WorkflowsPG.retentionTables)) {
|
|
18949
|
+
if (!entry.indexed || !policies[key]) continue;
|
|
18950
|
+
try {
|
|
18951
|
+
await this.#db.ensureIndex({
|
|
18952
|
+
indexName: `${prefix}mastra_${key}_retention_idx`,
|
|
18953
|
+
tableName: entry.table,
|
|
18954
|
+
column: entry.column
|
|
18955
|
+
});
|
|
18956
|
+
} catch (error) {
|
|
18957
|
+
this.logger?.warn?.(`Failed to create retention index for ${entry.table}:`, error);
|
|
18958
|
+
}
|
|
18959
|
+
}
|
|
18960
|
+
}
|
|
18961
|
+
/** Delete workflow run snapshots older than the `workflowSnapshot` policy's `maxAge`, batched. */
|
|
18962
|
+
async prune(policies, options) {
|
|
18963
|
+
await this.ensureRetentionIndexes(policies);
|
|
18964
|
+
const targets = resolveTargets({
|
|
18965
|
+
policies,
|
|
18966
|
+
descriptor: _WorkflowsPG.retentionTables,
|
|
18967
|
+
order: ["workflowSnapshot"]
|
|
18968
|
+
});
|
|
18969
|
+
return runPrune({ db: this.#db, domain: "workflows", targets, options });
|
|
18970
|
+
}
|
|
18224
18971
|
/**
|
|
18225
18972
|
* Creates custom user-defined indexes for this domain's tables.
|
|
18226
18973
|
*/
|
|
@@ -19334,7 +20081,7 @@ var PostgresStore = class extends MastraCompositeStore {
|
|
|
19334
20081
|
constructor(config) {
|
|
19335
20082
|
try {
|
|
19336
20083
|
validateConfig("PostgresStore", config);
|
|
19337
|
-
super({ id: config.id, name: "PostgresStore", disableInit: config.disableInit });
|
|
20084
|
+
super({ id: config.id, name: "PostgresStore", disableInit: config.disableInit, retention: config.retention });
|
|
19338
20085
|
this.schema = parseSqlIdentifier(config.schemaName || "public", "schema name");
|
|
19339
20086
|
if (isPoolConfig(config)) {
|
|
19340
20087
|
this.#pool = config.pool;
|