@mastra/duckdb 1.13.0-alpha.1 → 1.13.0-alpha.2

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-duckdb
 description: Documentation for @mastra/duckdb. Use when working with @mastra/duckdb APIs, configuration, or implementation.
 metadata:
   package: "@mastra/duckdb"
-  version: "1.13.0-alpha.1"
+  version: "1.13.0-alpha.2"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.13.0-alpha.1",
+  "version": "1.13.0-alpha.2",
   "package": "@mastra/duckdb",
   "exports": {},
   "modules": {}

package/dist/index.cjs

@@ -556,7 +556,7 @@ var ObservabilityStorageDuckDB = class extends _mastra_core_storage.Observabilit
 	async loadDelegate() {
 		if (this.delegate) return this.delegate;
 		if (this.unavailableError) return null;
-		if (!this.loadPromise) this.loadPromise = Promise.resolve().then(() => require("./observability-DNuNWZFX.cjs")).then(({ ObservabilityStorageDuckDB }) => {
+		if (!this.loadPromise) this.loadPromise = Promise.resolve().then(() => require("./observability-D4eFjsKJ.cjs")).then(({ ObservabilityStorageDuckDB }) => {
 			const delegate = new ObservabilityStorageDuckDB({ db: this.db });
 			this.delegate = delegate;
 			return delegate;

package/dist/index.js

@@ -555,7 +555,7 @@ var ObservabilityStorageDuckDB = class extends ObservabilityStorage {
 	async loadDelegate() {
 		if (this.delegate) return this.delegate;
 		if (this.unavailableError) return null;
-		if (!this.loadPromise) this.loadPromise = import("./observability-Bk_y3hof.js").then(({ ObservabilityStorageDuckDB }) => {
+		if (!this.loadPromise) this.loadPromise = import("./observability-DINJlR_a.js").then(({ ObservabilityStorageDuckDB }) => {
 			const delegate = new ObservabilityStorageDuckDB({ db: this.db });
 			this.delegate = delegate;
 			return delegate;

package/dist/{observability-DNuNWZFX.cjs → observability-D4eFjsKJ.cjs}

@@ -688,6 +688,33 @@ function jsonV(val) {
 	return require_features.DuckDBConnection.sqlValue(JSON.stringify(val));
 }
 /**
+* `span_events` JSON columns that can hold large payloads. Missing values are
+* stored as the JSON literal `null` instead of SQL NULL.
+*
+* DuckDB only reads the selected rows of a large-string column when the
+* column's validity mask is constant (no NULLs in the row group). Any SQL NULL
+* makes it fall back to reading the whole 2,048-row vector, which loads every
+* large string in it even when the query needs one row. Keeping these columns
+* NULL-free lets point lookups and page reads load only what they return.
+*/
+const SPAN_PAYLOAD_COLUMNS = [
+	"attributes",
+	"input",
+	"output",
+	"requestContext"
+];
+/** Stored in {@link SPAN_PAYLOAD_COLUMNS} in place of SQL NULL; read back as missing. */
+const JSON_NULL_PLACEHOLDER = "null";
+/** Like {@link jsonV}, but writes the JSON `null` placeholder instead of SQL NULL. */
+function payloadJsonV(val) {
+	if (val === null || val === void 0) return `'${JSON_NULL_PLACEHOLDER}'`;
+	return jsonV(val);
+}
+/** SQL for a payload column that maps the JSON `null` placeholder back to SQL NULL. */
+function payloadColumnSql(column) {
+	return `NULLIF(${column}, '${JSON_NULL_PLACEHOLDER}')`;
+}
+/**
 * Trim, drop blank, and dedupe tags before they are written, matching the
 * PostgreSQL and ClickHouse stores so tag predicates and discovery agree.
 */
@@ -2935,7 +2962,7 @@ const TRACE_SELECT = `
   r.entityId AS entityId,
   r.parentSpanId AS parentSpanId,
   r.metadata AS metadata,
-  r.input AS input,
+  ${payloadColumnSql("r.input")} AS input,
   r.threadId AS threadId,
   r.resourceId AS resourceId,
   r.startedAt AS startedAt,
@@ -3218,6 +3245,11 @@ function compileDuckDBTraceScope(relatedCollections, scope, timeRange, rootPredi
       GROUP BY r.traceId
       HAVING bool_or(r.timestamp >= CAST(? AS TIMESTAMP) AND r.timestamp < CAST(? AS TIMESTAMP))
         AND bool_or(${rowConditions.join("\n          AND ")})
+    )`,
+		`root_bounds AS (
+      SELECT min(timestamp) AS minTs, max(timestamp) AS maxTs
+      FROM span_events
+      WHERE parentSpanId IS NULL AND traceId IN (SELECT traceId FROM root_trace_ids)
     )`,
 		`root_events AS (
       SELECT
@@ -3228,6 +3260,8 @@ function compileDuckDBTraceScope(relatedCollections, scope, timeRange, rootPredi
         END AS startedAt
       FROM span_events
       WHERE parentSpanId IS NULL
+        AND timestamp >= (SELECT minTs FROM root_bounds)
+        AND timestamp <= (SELECT maxTs FROM root_bounds)
         AND traceId IN (SELECT traceId FROM root_trace_ids)
     )`,
 		`current_roots AS (
@@ -3250,7 +3284,15 @@ function compileDuckDBTraceScope(relatedCollections, scope, timeRange, rootPredi
     )`
 	];
 	if (relatedCollections.has("spans")) {
-		ctes.push(`current_span_rows AS (
+		ctes.push(`span_bounds AS (
+      -- Exact event-time range of the in-scope traces, from a narrow
+      -- (traceId, timestamp) pass. The join below cannot be pushed into the
+      -- scan, so the range keeps it from decompressing every span row.
+      SELECT min(timestamp) AS minTs, max(timestamp) AS maxTs
+      FROM span_events
+      WHERE traceId IN (SELECT traceId FROM root_scope)
+    ),
+    current_span_rows AS (
       SELECT
         e.*,
         coalesce(
@@ -3262,7 +3304,9 @@ function compileDuckDBTraceScope(relatedCollections, scope, timeRange, rootPredi
           ORDER BY CASE WHEN e.endedAt IS NULL THEN 1 ELSE 0 END ASC, e.cursorId DESC
         ) AS currentRank
       FROM span_events e
-      INNER JOIN root_scope roots ON roots.traceId = e.traceId${tenantWhere("e")}
+      INNER JOIN root_scope roots ON roots.traceId = e.traceId
+      WHERE e.timestamp >= (SELECT minTs FROM span_bounds)
+        AND e.timestamp <= (SELECT maxTs FROM span_bounds)${tenantConditions("e").map((condition) => `\n        AND ${condition}`).join("")}
     ),
     current_spans AS (
       SELECT
@@ -3812,7 +3856,7 @@ async function querySpans(db, plan) {
 		return `(${key} AND cursorId IS NOT DISTINCT FROM ? AND endedAt = ?)`;
 	}).join(" OR ");
 	const payloads = await query(`SELECT DISTINCT ${identities},
-    substring(CAST(input AS VARCHAR), 1, ${_mastra_core_storage.SPAN_QUERY_MAX_PREVIEW_CHARACTERS + 1}) AS inputPreview, substring(CAST(output AS VARCHAR), 1, ${_mastra_core_storage.SPAN_QUERY_MAX_PREVIEW_CHARACTERS + 1}) AS outputPreview
+    substring(CAST(${payloadColumnSql("input")} AS VARCHAR), 1, ${_mastra_core_storage.SPAN_QUERY_MAX_PREVIEW_CHARACTERS + 1}) AS inputPreview, substring(CAST(${payloadColumnSql("output")} AS VARCHAR), 1, ${_mastra_core_storage.SPAN_QUERY_MAX_PREVIEW_CHARACTERS + 1}) AS outputPreview
     FROM span_events WHERE ${payloadWhere}`, payloadValues);
 	const metricValues = [];
 	const metricWhere = keys(page, metricValues);
@@ -4067,13 +4111,15 @@ const COLUMNS_SQL = [
 	"error",
 	"requestContext"
 ].join(", ");
+const PAYLOAD_COLUMN_SET = new Set(SPAN_PAYLOAD_COLUMNS);
 /**
 * Reconstruction query uses `arg_max(field, timestamp) FILTER (WHERE field IS NOT NULL)`
 * so that the final end event supplies the terminal span fields without wiping
-* stable values emitted on the start event.
+* stable values emitted on the start event. Payload columns also skip the JSON
+* `null` placeholder, which stands in for a missing value.
 */
 function argMaxNonNull(col) {
-	return `arg_max(${col}, timestamp) FILTER (WHERE ${col} IS NOT NULL) as ${col}`;
+	return `arg_max(${col}, timestamp) FILTER (WHERE ${col} IS NOT NULL${PAYLOAD_COLUMN_SET.has(col) ? ` AND ${col} <> '${JSON_NULL_PLACEHOLDER}'` : ""}) as ${col}`;
 }
 const SPAN_RECONSTRUCT_SELECT = `
   SELECT
@@ -4184,28 +4230,55 @@ function buildPostAggReconstructSelect(postAgg, orderByField) {
 `;
 }
 /**
-* Reconstruct spans for the `(traceId, spanId)` pairs selected by `anchorCte`,
-* scanning only events at/after the CTE's earliest `anchorStartedAt`.
+* Reconstruct the spans of one list page, reading payload columns only for
+* the page's own events.
+*
+* Two queries:
+*   1. `anchorSql` (a `WITH ...` prefix defining `anchorCte`) picks the page's
+*      `(traceId, spanId)` anchors, and a narrow pass over `traceId`, `spanId`
+*      and `timestamp` finds the exact time range of their events.
+*   2. Reconstruction scans only that range for exactly those anchors.
 *
-* The `(traceId, spanId) IN (subquery)` semi-join alone cannot be pushed into
-* the table scan, so without a bound DuckDB decompresses every column of the
-* entire table just to emit one page of spans. The time bound is a plain range
-* predicate that zone maps can prune on (insertion order tracks event time).
+* The `(traceId, spanId) IN (...)` semi-join cannot be pushed into the table
+* scan, so without a range DuckDB decompresses every payload column of every
+* row it reads. A lower bound alone (the earliest anchor) prunes nothing for
+* the oldest pages, which then read the whole table and can run out of memory.
+* Both bounds are passed as literal parameters so they always reach the scan
+* as plain range filters that zone maps can prune on.
 *
-* Correctness: span events are only 'start' (timestamp = startedAt) and 'end'
-* (timestamp = endedAt >= startedAt), so every event of an anchored span has
-* timestamp >= its start-row timestamp >= min(anchorStartedAt). An empty
-* anchor set makes the bound NULL, which matches the empty IN-list result.
+* Anchors and bounds are frozen after the first query, so spans written in
+* between cannot change which rows are on the page. An event a page span
+* receives in between is left for the next read, as if the page had been
+* read a moment earlier.
 */
-function reconstructForAnchors(reconstructSelect, anchorCte) {
-	return `
-    ${reconstructSelect}
-    WHERE timestamp >= (SELECT min(anchorStartedAt) FROM ${anchorCte})
-      AND (traceId, spanId) IN (SELECT traceId, spanId FROM ${anchorCte})
-    GROUP BY traceId, spanId`;
+async function reconstructAnchoredPage(db, anchorSql, anchorCte, params, reconstructSelect, orderBySql) {
+	const anchors = await db.query(`
+      ${anchorSql}
+      SELECT anchors.traceId, anchors.spanId, bounds.minTs, bounds.maxTs
+      FROM ${anchorCte} AS anchors,
+        (
+          SELECT min(timestamp) AS minTs, max(timestamp) AS maxTs
+          FROM span_events
+          WHERE (traceId, spanId) IN (SELECT traceId, spanId FROM ${anchorCte})
+        ) AS bounds
+    `, params);
+	if (anchors.length === 0) return [];
+	const { minTs, maxTs } = anchors[0];
+	const anchorValues = anchors.map(() => "(?, ?)").join(", ");
+	return db.query(`
+      ${reconstructSelect}
+      WHERE timestamp >= ? AND timestamp <= ?
+        AND (traceId, spanId) IN (SELECT * FROM (VALUES ${anchorValues}))
+      GROUP BY traceId, spanId
+      ${orderBySql}
+    `, [
+		minTs,
+		maxTs,
+		...anchors.flatMap((anchor) => [anchor.traceId, anchor.spanId])
+	]);
 }
 /**
-* Same time-bound trick as {@link reconstructForAnchors}, but with the bound
+* Same time-bound trick as {@link reconstructAnchoredPage}, but with the bound
 * precomputed in JS and passed as a `?` parameter. The delta-poll query shape
 * references its candidate CTE multiple times, which makes DuckDB materialize
 * it and lose the dynamic-filter pushdown a scalar subquery bound relies on —
@@ -4439,15 +4512,15 @@ function toValuesTuple(row) {
 		v(row.environment),
 		v(row.source),
 		v(row.serviceName),
-		jsonV(row.attributes),
+		payloadJsonV(row.attributes),
 		jsonV(row.metadata),
 		jsonV(row.tags),
 		jsonV(row.scope),
 		jsonV(row.links),
-		jsonV(row.input),
-		jsonV(row.output),
+		payloadJsonV(row.input),
+		payloadJsonV(row.output),
 		jsonV(row.error),
-		jsonV(row.requestContext)
+		payloadJsonV(row.requestContext)
 	].join(", ");
 }
 async function insertSpanEvents(db, rows) {
@@ -4661,7 +4734,7 @@ async function listTraceRows(db, args, reconstructSelect, mapRow, toSpans) {
 			},
 			spans: []
 		};
-		const pageSql = `
+		const spans = (await reconstructAnchoredPage(db, `
       WITH page_roots AS (
         SELECT traceId, spanId, min(timestamp) AS anchorStartedAt
         FROM span_events AS ${outerAlias}
@@ -4670,14 +4743,11 @@ async function listTraceRows(db, args, reconstructSelect, mapRow, toSpans) {
         ${prefilterOrderBy}
         LIMIT ? OFFSET ?
       )
-      ${reconstructForAnchors(reconstructSelect, "page_roots")}
-      ${buildOrderByClause(orderBy)}, traceId, spanId
-    `;
-		const spans = (await db.query(pageSql, [
+    `, "page_roots", [
 			...prefilterParams,
 			perPage,
 			offset
-		])).map((row) => mapRow(row));
+		], reconstructSelect, `${buildOrderByClause(orderBy)}, traceId, spanId`)).map((row) => mapRow(row));
 		return {
 			pagination: {
 				total,
@@ -4726,20 +4796,17 @@ async function listTraceRows(db, args, reconstructSelect, mapRow, toSpans) {
 		},
 		spans: []
 	};
-	const dataSql = `
+	const spans = (await reconstructAnchoredPage(db, `
     ${cteSql},
     page_roots AS (
       SELECT traceId, spanId, startedAt AS anchorStartedAt
       FROM root_spans ${postAggWhere} ${orderByClause} ${paginationClause}
     )
-    ${reconstructForAnchors(reconstructSelect, "page_roots")}
-    ${orderByClause}
-  `;
-	const spans = (await db.query(dataSql, [
+  `, "page_roots", [
 		...prefilterParams,
 		...postAggParams,
 		...paginationParams
-	])).map((row) => mapRow(row));
+	], reconstructSelect, orderByClause)).map((row) => mapRow(row));
 	return {
 		pagination: {
 			total,
@@ -5043,7 +5110,7 @@ async function listBranches(db, args) {
 			branches: [],
 			...deltaPollingFeatureEnabled() ? { deltaCursor: currentDeltaCursor } : {}
 		};
-		const pageSql = `
+		const spans = (await reconstructAnchoredPage(db, `
       WITH page_anchors AS (
         SELECT traceId, spanId, min(timestamp) AS anchorStartedAt
         FROM span_events AS ${outerAlias}
@@ -5052,14 +5119,11 @@ async function listBranches(db, args) {
         ${prefilterOrderBy}
         LIMIT ? OFFSET ?
       )
-      ${reconstructForAnchors(SPAN_RECONSTRUCT_SELECT, "page_anchors")}
-      ${buildOrderByClause(orderBy)}, traceId, spanId
-    `;
-		const spans = (await db.query(pageSql, [
+    `, "page_anchors", [
 			...prefilterParams,
 			perPage,
 			offset
-		])).map((row) => rowToSpanRecord(row));
+		], SPAN_RECONSTRUCT_SELECT, `${buildOrderByClause(orderBy)}, traceId, spanId`)).map((row) => rowToSpanRecord(row));
 		return {
 			pagination: {
 				total,
@@ -5106,20 +5170,17 @@ async function listBranches(db, args) {
 		branches: [],
 		...deltaPollingFeatureEnabled() ? { deltaCursor: currentDeltaCursor } : {}
 	};
-	const dataSql = `
+	const spans = (await reconstructAnchoredPage(db, `
     ${cteSql},
     page_anchors AS (
       SELECT traceId, spanId, startedAt AS anchorStartedAt
       FROM branch_anchors ${postAggWhere} ${orderByClause} ${paginationClause}
     )
-    ${reconstructForAnchors(SPAN_RECONSTRUCT_SELECT, "page_anchors")}
-    ${orderByClause}
-  `;
-	const spans = (await db.query(dataSql, [
+  `, "page_anchors", [
 		...prefilterParams,
 		...postAggParams,
 		...paginationParams
-	])).map((row) => rowToSpanRecord(row));
+	], SPAN_RECONSTRUCT_SELECT, orderByClause)).map((row) => rowToSpanRecord(row));
 	return {
 		pagination: {
 			total,
@@ -5543,4 +5604,4 @@ var ObservabilityStorageDuckDB = class ObservabilityStorageDuckDB extends _mastr
 //#endregion
 exports.ObservabilityStorageDuckDB = ObservabilityStorageDuckDB;
 
-//# sourceMappingURL=observability-DNuNWZFX.cjs.map
+//# sourceMappingURL=observability-D4eFjsKJ.cjs.map