@mastra/pg 1.26.0-alpha.1 → 1.26.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/reference-storage-retention.md +56 -4
- package/dist/index.cjs +38 -127
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +39 -128
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-query.d.ts.map +1 -1
- package/dist/storage/retention.d.ts +5 -58
- package/dist/storage/retention.d.ts.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "crypto";
|
|
2
2
|
import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
|
|
3
3
|
import * as coreStorage from "@mastra/core/storage";
|
|
4
|
-
import { AgentsStorage, BRANCH_SPAN_TYPES, BackgroundTasksStorage, BlobStore, ChannelsStorage, DATASETS_SCHEMA, DATASET_ITEMS_SCHEMA, DATASET_VERSIONS_SCHEMA, DatasetsStorage, EXPERIMENTS_SCHEMA, EXPERIMENT_RESULTS_SCHEMA, EntityType, ExperimentsStorage, FactoryStorage, FavoritesStorage, KNOWLEDGE_ACTIVITY_SCHEMA, KNOWLEDGE_CURSORS_SCHEMA, KNOWLEDGE_MENTIONS_SCHEMA, KNOWLEDGE_NODES_SCHEMA, KNOWLEDGE_RECORDS_SCHEMA, KNOWLEDGE_SEMANTIC_OUTBOX_SCHEMA, KnowledgeConflictError, KnowledgeNotFoundError, KnowledgeStorage, MCPClientsStorage, MCPServersStorage, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, NotificationsStorage, OBSERVATIONAL_MEMORY_TABLE_SCHEMA, ObservabilityStorage, PromptBlocksStorage, SchedulesStorage, ScorerDefinitionsStorage, ScoresStorage, SkillsStorage, TABLE_AGENTS, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_CHANNEL_CONFIG, TABLE_CHANNEL_INSTALLATIONS, TABLE_CONFIGS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_KNOWLEDGE_ACTIVITY, TABLE_KNOWLEDGE_CURSORS, TABLE_KNOWLEDGE_MENTIONS, TABLE_KNOWLEDGE_NODES, TABLE_KNOWLEDGE_RECORDS, TABLE_KNOWLEDGE_SEMANTIC_OUTBOX, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, ThreadStateStorage, ToolProviderConnectionsStorage, TraceStatus, UniqueViolationError, WorkflowDefinitionsStorage, WorkflowsStorage, WorkspacesStorage, assertKnowledgeCeilingRaised, assertKnowledgeScopeWithinCeiling, calculatePagination, canonicalizeKnowledgeScope, createKnowledgeUlid, createStorageErrorId, createVectorErrorId, ensureDate, getDefaultValue, getSqlType, hasErrorCode, isKnowledgeScopeVisible, knowledgeScopeKey, knowledgeSemanticDocumentId, knowledgeSemanticIdempotencyKey, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, matchesExpectedWorkflowStatus, mergeWorkflowStepResult, normalizePerPage, normalizeScheduleTarget, parseDuration, parseKnowledgeNodeCursor, parseKnowledgeWikilinks, safelyParseJSON, storageMessageMatchesMetadataFilter, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
|
|
4
|
+
import { AgentsStorage, BRANCH_SPAN_TYPES, BackgroundTasksStorage, BlobStore, ChannelsStorage, DATASETS_SCHEMA, DATASET_ITEMS_SCHEMA, DATASET_VERSIONS_SCHEMA, DatasetsStorage, EXPERIMENTS_SCHEMA, EXPERIMENT_RESULTS_SCHEMA, EntityType, ExperimentsStorage, FactoryStorage, FavoritesStorage, KNOWLEDGE_ACTIVITY_SCHEMA, KNOWLEDGE_CURSORS_SCHEMA, KNOWLEDGE_MENTIONS_SCHEMA, KNOWLEDGE_NODES_SCHEMA, KNOWLEDGE_RECORDS_SCHEMA, KNOWLEDGE_SEMANTIC_OUTBOX_SCHEMA, KnowledgeConflictError, KnowledgeNotFoundError, KnowledgeStorage, MCPClientsStorage, MCPServersStorage, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, NotificationsStorage, OBSERVATIONAL_MEMORY_TABLE_SCHEMA, ObservabilityStorage, PromptBlocksStorage, SchedulesStorage, ScorerDefinitionsStorage, ScoresStorage, SkillsStorage, TABLE_AGENTS, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_CHANNEL_CONFIG, TABLE_CHANNEL_INSTALLATIONS, TABLE_CONFIGS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_KNOWLEDGE_ACTIVITY, TABLE_KNOWLEDGE_CURSORS, TABLE_KNOWLEDGE_MENTIONS, TABLE_KNOWLEDGE_NODES, TABLE_KNOWLEDGE_RECORDS, TABLE_KNOWLEDGE_SEMANTIC_OUTBOX, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, ThreadStateStorage, ToolProviderConnectionsStorage, TraceStatus, UniqueViolationError, WorkflowDefinitionsStorage, WorkflowsStorage, WorkspacesStorage, assertKnowledgeCeilingRaised, assertKnowledgeScopeWithinCeiling, calculatePagination, canonicalizeKnowledgeScope, createKnowledgeUlid, createStorageErrorId, createVectorErrorId, ensureDate, executeRetentionPrune, getDefaultValue, getSqlType, hasErrorCode, isKnowledgeScopeVisible, knowledgeScopeKey, knowledgeSemanticDocumentId, knowledgeSemanticIdempotencyKey, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, matchesExpectedWorkflowStatus, mergeWorkflowStepResult, normalizePerPage, normalizeScheduleTarget, parseDuration, parseKnowledgeNodeCursor, parseKnowledgeWikilinks, resolveRetentionTargets, retentionCutoffMs, runRetentionBatches, safelyParseJSON, storageMessageMatchesMetadataFilter, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
|
|
5
5
|
import { parseFieldKey, parseSqlIdentifier } from "@mastra/core/utils";
|
|
6
6
|
import { MastraVector, validateTopK, validateUpsertInput } from "@mastra/core/vector";
|
|
7
7
|
import { Mutex } from "async-mutex";
|
|
@@ -4903,137 +4903,31 @@ var AgentsPG = class AgentsPG extends AgentsStorage {
|
|
|
4903
4903
|
};
|
|
4904
4904
|
//#endregion
|
|
4905
4905
|
//#region src/storage/retention.ts
|
|
4906
|
-
const DEFAULT_BATCH_SIZE = 1e3;
|
|
4907
|
-
async function sleep(ms, signal) {
|
|
4908
|
-
if (ms <= 0 || signal?.aborted) return;
|
|
4909
|
-
await new Promise((resolve) => {
|
|
4910
|
-
const timer = setTimeout(() => {
|
|
4911
|
-
signal?.removeEventListener("abort", onAbort);
|
|
4912
|
-
resolve();
|
|
4913
|
-
}, ms);
|
|
4914
|
-
const onAbort = () => {
|
|
4915
|
-
clearTimeout(timer);
|
|
4916
|
-
resolve();
|
|
4917
|
-
};
|
|
4918
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
4919
|
-
});
|
|
4920
|
-
}
|
|
4921
|
-
/**
|
|
4922
|
-
* Convert a policy's `maxAge` into a cutoff bound matching the anchor's storage
|
|
4923
|
-
* type: a `Date` for `timestamptz` columns (pg compares timezone-aware), or a
|
|
4924
|
-
* raw millisecond number for `bigint` epoch-ms columns.
|
|
4925
|
-
*/
|
|
4926
4906
|
function cutoffFor(policy, anchorType, now = Date.now()) {
|
|
4927
|
-
const cutoffMs =
|
|
4907
|
+
const cutoffMs = retentionCutoffMs(policy, now);
|
|
4928
4908
|
return anchorType === "epoch-ms" ? cutoffMs : new Date(cutoffMs);
|
|
4929
4909
|
}
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
|
|
4938
|
-
|
|
4939
|
-
|
|
4940
|
-
|
|
4941
|
-
|
|
4942
|
-
|
|
4943
|
-
|
|
4944
|
-
};
|
|
4945
|
-
if (options?.maxBatches !== void 0 && batches >= options.maxBatches) return {
|
|
4946
|
-
deleted,
|
|
4947
|
-
done: false
|
|
4948
|
-
};
|
|
4949
|
-
let limit = batchSize;
|
|
4950
|
-
if (options?.maxRows !== void 0) {
|
|
4951
|
-
const remaining = options.maxRows - deleted;
|
|
4952
|
-
if (remaining <= 0) return {
|
|
4953
|
-
deleted,
|
|
4954
|
-
done: false
|
|
4955
|
-
};
|
|
4956
|
-
limit = Math.min(limit, remaining);
|
|
4957
|
-
}
|
|
4958
|
-
const affected = await deleteBatch(limit);
|
|
4959
|
-
deleted += affected;
|
|
4960
|
-
batches += 1;
|
|
4961
|
-
if (affected < limit) return {
|
|
4962
|
-
deleted,
|
|
4963
|
-
done: true
|
|
4964
|
-
};
|
|
4965
|
-
if (options?.pauseMs) await sleep(options.pauseMs, options.signal);
|
|
4966
|
-
}
|
|
4967
|
-
}
|
|
4968
|
-
/**
|
|
4969
|
-
* Runs the bounded, batched, cancellable delete loop for a set of tables in the
|
|
4970
|
-
* given order (callers pass children before parents for cascade-safe pruning),
|
|
4971
|
-
* and returns one {@link PruneResult} per table.
|
|
4972
|
-
*
|
|
4973
|
-
* The loop:
|
|
4974
|
-
* - deletes in chunks of `batchSize` (default 1000), each its own statement;
|
|
4975
|
-
* - stops a table's loop when a batch deletes fewer rows than requested (drained),
|
|
4976
|
-
* or when `maxBatches`/`maxRows` is hit, or the `signal` aborts — the latter
|
|
4977
|
-
* three leave `done: false` so the caller can resume;
|
|
4978
|
-
* - pauses `pauseMs` between batches when set, to avoid starving live traffic.
|
|
4979
|
-
*
|
|
4980
|
-
* `prune()` only deletes rows; it never reclaims disk. PostgreSQL reuses freed
|
|
4981
|
-
* space (dead tuples) via autovacuum on subsequent writes, so tables stop
|
|
4982
|
-
* growing. Returning disk to the OS (e.g. `VACUUM FULL`) is left to the operator.
|
|
4983
|
-
*/
|
|
4984
|
-
async function runPrune({ db, domain, targets, options }) {
|
|
4985
|
-
const results = [];
|
|
4986
|
-
const now = Date.now();
|
|
4987
|
-
for (const target of targets) {
|
|
4988
|
-
if (options?.signal?.aborted) {
|
|
4989
|
-
results.push({
|
|
4990
|
-
domain,
|
|
4991
|
-
table: target.table,
|
|
4992
|
-
deleted: 0,
|
|
4993
|
-
done: false
|
|
4994
|
-
});
|
|
4995
|
-
continue;
|
|
4996
|
-
}
|
|
4997
|
-
const cutoff = cutoffFor(target.policy, target.anchorType, now);
|
|
4998
|
-
const { deleted, done } = await runBatchedDelete({
|
|
4999
|
-
deleteBatch: (limit) => db.pruneBatch({
|
|
5000
|
-
tableName: target.table,
|
|
5001
|
-
column: target.column,
|
|
5002
|
-
cutoff,
|
|
5003
|
-
limit
|
|
5004
|
-
}),
|
|
5005
|
-
batchSize: target.policy.batchSize ?? DEFAULT_BATCH_SIZE,
|
|
5006
|
-
options
|
|
5007
|
-
});
|
|
5008
|
-
results.push({
|
|
5009
|
-
domain,
|
|
5010
|
-
table: target.table,
|
|
5011
|
-
deleted,
|
|
5012
|
-
done
|
|
5013
|
-
});
|
|
5014
|
-
}
|
|
5015
|
-
return results;
|
|
4910
|
+
const runBatchedDelete = runRetentionBatches;
|
|
4911
|
+
function runPrune({ db, domain, targets, options }) {
|
|
4912
|
+
return executeRetentionPrune({
|
|
4913
|
+
domain,
|
|
4914
|
+
targets,
|
|
4915
|
+
options,
|
|
4916
|
+
cutoffFor: (target, now) => cutoffFor(target.policy, target.anchorType ?? "timestamp", now),
|
|
4917
|
+
deleteBatch: (target, cutoff, limit) => db.pruneBatch({
|
|
4918
|
+
tableName: target.table,
|
|
4919
|
+
column: target.column,
|
|
4920
|
+
cutoff,
|
|
4921
|
+
limit
|
|
4922
|
+
})
|
|
4923
|
+
});
|
|
5016
4924
|
}
|
|
5017
|
-
/**
|
|
5018
|
-
* Resolve a domain's `{ tableKey: policy }` map plus its descriptor into an
|
|
5019
|
-
* ordered list of {@link PruneTarget}s. `order` lists table keys children-first
|
|
5020
|
-
* so cascade-dependent rows are removed before their parents. Table keys not in
|
|
5021
|
-
* `policies` are skipped (unset = keep forever).
|
|
5022
|
-
*/
|
|
5023
4925
|
function resolveTargets({ policies, descriptor, order }) {
|
|
5024
|
-
|
|
5025
|
-
|
|
5026
|
-
|
|
5027
|
-
|
|
5028
|
-
|
|
5029
|
-
targets.push({
|
|
5030
|
-
table: entry.table,
|
|
5031
|
-
column: entry.column,
|
|
5032
|
-
anchorType: entry.anchorType ?? "timestamp",
|
|
5033
|
-
policy
|
|
5034
|
-
});
|
|
5035
|
-
}
|
|
5036
|
-
return targets;
|
|
4926
|
+
return resolveRetentionTargets({
|
|
4927
|
+
policies,
|
|
4928
|
+
descriptor,
|
|
4929
|
+
order
|
|
4930
|
+
});
|
|
5037
4931
|
}
|
|
5038
4932
|
//#endregion
|
|
5039
4933
|
//#region src/storage/domains/background-tasks/index.ts
|
|
@@ -17291,6 +17185,11 @@ const FEEDBACK_FIELDS = {
|
|
|
17291
17185
|
const TRACE_SELECT = `
|
|
17292
17186
|
r."traceId" AS "traceId",
|
|
17293
17187
|
r."spanId" AS "rootSpanId",
|
|
17188
|
+
r."name" AS "name",
|
|
17189
|
+
r."entityId" AS "entityId",
|
|
17190
|
+
r."parentSpanId" AS "parentSpanId",
|
|
17191
|
+
r."metadata" AS "metadata",
|
|
17192
|
+
r."input" AS "input",
|
|
17294
17193
|
r."threadId" AS "threadId",
|
|
17295
17194
|
r."resourceId" AS "resourceId",
|
|
17296
17195
|
r."startedAt" AS "startedAt",
|
|
@@ -17770,6 +17669,11 @@ function isPostgresStatementTimeout(error) {
|
|
|
17770
17669
|
const candidate = error;
|
|
17771
17670
|
return candidate.code === "57014" && String(candidate.message ?? "").includes("statement timeout");
|
|
17772
17671
|
}
|
|
17672
|
+
function isPostgresResourceLimit(error) {
|
|
17673
|
+
if (!error || typeof error !== "object") return false;
|
|
17674
|
+
const candidate = error;
|
|
17675
|
+
return candidate.code === "53200" || candidate.code === "53400";
|
|
17676
|
+
}
|
|
17773
17677
|
async function runWithPostgresTraceQueryTimeout(client, timeoutMs, execute) {
|
|
17774
17678
|
const resolvedTimeoutMs = coreStorage.resolveTraceQueryTimeoutMs(timeoutMs);
|
|
17775
17679
|
try {
|
|
@@ -17779,6 +17683,7 @@ async function runWithPostgresTraceQueryTimeout(client, timeoutMs, execute) {
|
|
|
17779
17683
|
});
|
|
17780
17684
|
} catch (error) {
|
|
17781
17685
|
if (isPostgresStatementTimeout(error)) throw new coreStorage.TraceQueryExecutionError();
|
|
17686
|
+
if (isPostgresResourceLimit(error)) throw new coreStorage.TraceQueryResourceLimitError();
|
|
17782
17687
|
throw error;
|
|
17783
17688
|
}
|
|
17784
17689
|
}
|
|
@@ -17823,6 +17728,12 @@ async function queryTraces(client, schema, plan, timeoutMs) {
|
|
|
17823
17728
|
const traces = visibleRows.map((row) => ({
|
|
17824
17729
|
traceId: String(row.traceId),
|
|
17825
17730
|
rootSpanId: String(row.rootSpanId),
|
|
17731
|
+
name: row.name,
|
|
17732
|
+
entityId: row.entityId ?? null,
|
|
17733
|
+
parentSpanId: row.parentSpanId ?? null,
|
|
17734
|
+
createdAt: asIsoTimestamp$1(row.startedAt),
|
|
17735
|
+
metadata: row.metadata ?? null,
|
|
17736
|
+
inputPreview: coreStorage.buildInputPreview(row.input) ?? null,
|
|
17826
17737
|
threadId: row.threadId == null ? null : String(row.threadId),
|
|
17827
17738
|
resourceId: row.resourceId == null ? null : String(row.resourceId),
|
|
17828
17739
|
startedAt: asIsoTimestamp$1(row.startedAt),
|
|
@@ -18570,7 +18481,7 @@ async function dangerouslyClearTracing(client, schema) {
|
|
|
18570
18481
|
* Use it through `MastraCompositeStore` with a dedicated Postgres connection.
|
|
18571
18482
|
*/
|
|
18572
18483
|
function wrapError(op, error, details) {
|
|
18573
|
-
if (error instanceof MastraError || error instanceof coreStorage.TraceQueryExecutionError) throw error;
|
|
18484
|
+
if (error instanceof MastraError || error instanceof coreStorage.TraceQueryExecutionError || error instanceof coreStorage.TraceQueryResourceLimitError) throw error;
|
|
18574
18485
|
throw new MastraError({
|
|
18575
18486
|
id: createStorageErrorId("PG", op, "FAILED"),
|
|
18576
18487
|
domain: ErrorDomain.STORAGE,
|