@gaganref/confect-server 10.3.1-fork.8 → 10.3.1-fork.80

package/dist/QueryStream.js

@@ -11,17 +11,17 @@ import { admittedByLower, admittedByUpper, forLayout, fromParsed, intersect, int
 import { apply, equalityPrefixLength, fromBounds, toBounds as toBounds$1 } from "./QueryStreamIndexRange.js";
 import { EmptyInitialPageError, InvalidPageSizeError, Range, Start, finish, initial, parseRequest, record } from "./QueryStreamPagination.js";
 import { InvalidReadLimitError, Limits, QueryStreamReadBudget, make as make$1 } from "./QueryStreamReadBudget.js";
+import { ConvexError } from "convex/values";
 import * as Effect from "effect/Effect";
+import * as Data from "effect/Data";
+import * as Predicate from "effect/Predicate";
+import * as Array from "effect/Array";
 import * as Schema from "effect/Schema";
 import { dual, identity, pipe } from "effect/Function";
 import * as Match from "effect/Match";
 import * as Option from "effect/Option";
-import { ConvexError } from "convex/values";
-import * as Array from "effect/Array";
 import * as Order from "effect/Order";
-import * as Predicate from "effect/Predicate";
 import * as Result from "effect/Result";
-import * as Data from "effect/Data";
 import * as Stream from "effect/Stream";
 import { pipeArguments } from "effect/Pipeable";
 import * as Chunk from "effect/Chunk";
@@ -32,6 +32,8 @@ import * as String from "effect/String";
 import * as Tuple from "effect/Tuple";
 //#region src/QueryStream.ts
 var QueryStream_exports = /* @__PURE__ */ __exportAll({
+	DEFAULT_MAXIMUM_ROWS_READ: () => DEFAULT_MAXIMUM_ROWS_READ,
+	DeferredStreamMismatchError: () => DeferredStreamMismatchError,
 	Element: () => Element,
 	EmptyInitialPageError: () => EmptyInitialPageError,
 	IncompatibleStreamsError: () => IncompatibleStreamsError,
@@ -44,6 +46,8 @@ var QueryStream_exports = /* @__PURE__ */ __exportAll({
 	QueryStream: () => QueryStream,
 	ReadBudgetExceededError: () => ReadBudgetExceededError,
 	TypeId: () => TypeId,
+	collect: () => collect,
+	defer: () => defer,
 	distinct: () => distinct,
 	empty: () => empty,
 	filter: () => filter,
@@ -61,7 +65,15 @@ var QueryStream_exports = /* @__PURE__ */ __exportAll({
 	unique: () => unique
 });
 /**
-* A page exhausted its read budget without a safe logical continuation.
+* Default physical row budget for `paginate` and `collect`. Supply an explicit
+* `maximumRowsRead` to override it. Plain Effect Stream consumers use their
+* enclosing budget and remain unlimited when none is provided.
+*
+* @experimental
+*/
+const DEFAULT_MAXIMUM_ROWS_READ = 2e3;
+/**
+* A query stream exhausted its read budget before its consumer could finish.
 *
 * @experimental
 */
@@ -70,7 +82,7 @@ var ReadBudgetExceededError = class extends Schema.TaggedError()("ReadBudgetExce
 	bytesRead: Schema.Natural
 }) {
 	get message() {
-		return "The read budget was exhausted before a safe page boundary; increase the budget or simplify the query";
+		return "The read budget was exhausted before the query stream completed; increase the budget or simplify the query";
 	}
 };
 /**
@@ -234,6 +246,40 @@ const empty = () => (keyLayout, orderDirection = "asc") => {
 	return make(orderDirection);
 };
 /**
+* A deferred stream's resolved direction or layout differs from its
+* declaration. The check runs on consumption, before the resolved source is
+* pulled.
+*
+* @experimental
+*/
+var DeferredStreamMismatchError = class extends Data.TaggedError("DeferredStreamMismatchError") {};
+/**
+* Defer an effect that checks access or constructs a query stream until it is
+* pulled. Supply its layout and direction without reading documents. This lets
+* `flatMap` build and narrow an inner stream before running its checks.
+*
+* The effect runs afresh on each consumption, seek, or reversal. Failures and
+* service requirements are preserved. Its stream must match the declared layout
+* and direction, or consumption dies with `DeferredStreamMismatchError`.
+*
+* @experimental
+*/
+const defer = (options, make) => {
+	const checked = Effect.map(make, (stream) => {
+		if (stream.orderDirection !== options.orderDirection || !Equivalence(stream.keyLayout, options.keyLayout)) throw new DeferredStreamMismatchError({
+			expectedOrderDirection: options.orderDirection,
+			actualOrderDirection: stream.orderDirection,
+			expectedKeyLayout: options.keyLayout,
+			actualKeyLayout: stream.keyLayout
+		});
+		return stream;
+	});
+	return new QueryStream(options.orderDirection, options.keyLayout, Stream.unwrap(Effect.map(checked, (stream) => stream.annotated)), void 0, (bounds) => defer(options, Effect.map(checked, (stream) => narrowByParsedBounds(stream, bounds))), () => defer({
+		...options,
+		orderDirection: flip(options.orderDirection)
+	}, Effect.map(checked, reverse)));
+};
+/**
 * Create a reusable query stream from an index-query description. Prefer
 * `reader.table(...).stream(...)` for inferred document and ordering types.
 *
@@ -297,13 +343,21 @@ const fillMergeSource = (source) => MergeSource.$match(source, {
 	Exhausted: () => Effect.succeed(source),
 	BudgetLimited: () => Effect.succeed(source)
 });
+const consumeMergeHead = (source) => source._tag === "Buffered" ? Option.match(Chunk.head(source.tail), {
+	onNone: () => MergeSource.NeedsPull({ pull: source.pull }),
+	onSome: (head) => MergeSource.Buffered({
+		pull: source.pull,
+		head,
+		tail: Chunk.drop(source.tail, 1)
+	})
+}) : source;
 /**
 * One step of the k-way merge as a pure unfold: fill every source, emit the
 * earliest head (ties go to the earliest source, keeping the merge stable), and
 * return the sources with that head consumed. `undefined` when every source is
 * exhausted or an input stopped before its next key was known.
 */
-const mergeStep = (keyOrder) => (sources) => Effect.gen(function* () {
+const mergeStep = (keyOrder, dedupe) => (sources) => Effect.gen(function* () {
 	const readBudget = yield* QueryStreamReadBudget;
 	const filled = yield* Effect.forEach(sources, fillMergeSource, { concurrency: (yield* readBudget.isUnlimited) ? "unbounded" : 1 });
 	if (filled.some((source) => MergeSource.$is("BudgetLimited")(source))) return void 0;
@@ -321,23 +375,35 @@ const mergeStep = (keyOrder) => (sources) => Effect.gen(function* () {
 			})) : best
 		})
 	}));
-	return Option.getOrUndefined(Option.map(earliest, ({ index, element }) => Tuple.make(element, Array.map(filled, (source, sourceIndex) => sourceIndex === index && source._tag === "Buffered" ? Option.match(Chunk.head(source.tail), {
-		onNone: () => MergeSource.NeedsPull({ pull: source.pull }),
-		onSome: (head) => MergeSource.Buffered({
-			pull: source.pull,
-			head,
-			tail: Chunk.drop(source.tail, 1)
-		})
-	}) : source))));
+	if (Option.isNone(earliest)) return void 0;
+	if (!dedupe) {
+		const { index, element } = earliest.value;
+		return Tuple.make(element, Array.map(filled, (source, sourceIndex) => sourceIndex === index ? consumeMergeHead(source) : source));
+	}
+	const keyValues = earliest.value.element.keyValues;
+	const matches = (source) => source._tag === "Buffered" && keyOrder(source.head.keyValues, keyValues) === 0;
+	let representative = earliest.value;
+	let remaining = filled;
+	for (;;) {
+		representative = Array.reduce(remaining, representative, (best, source, index) => source._tag === "Buffered" && matches(source) && Option.isSome(source.head.doc) && (Option.isNone(best.element.doc) || index < best.index) ? new MergeCandidate({
+			index,
+			element: source.head
+		}) : best);
+		remaining = yield* Effect.forEach(Array.map(remaining, (source) => matches(source) ? consumeMergeHead(source) : source), fillMergeSource, { concurrency: (yield* readBudget.isUnlimited) ? "unbounded" : 1 });
+		if (remaining.some((source) => MergeSource.$is("BudgetLimited")(source))) return void 0;
+		if (!remaining.some(matches)) return Tuple.make(representative.element, remaining);
+	}
 });
 /**
 * Combine streams sharing a key layout into one stream in key order. Use this
 * to query several index ranges at once, such as notes by multiple roles.
 *
 * The merge emits the smallest next key when ascending and the largest when
-* descending. Unlike `Stream.merge`, it does not interleave by arrival time.
-* Overlapping inputs are not deduplicated: a document matched twice appears
-* twice. Supply at least one input; use `empty` for an empty dynamic list.
+* descending. Unlike `Stream.merge`, it does not interleave by arrival time. By
+* default, a document matched twice appears twice. Pass `{ dedupe: true }` to
+* coalesce equal complete keys, including ID tiebreakers. Resolving a group can
+* prefetch the next key from each input; all such reads count against the
+* shared budget. Supply at least one input; use `empty` for an empty list.
 *
 * Inputs need compatible document types and identical key layouts and
 * directions. TypeScript rejects known mismatches; runtime mismatches throw
@@ -346,7 +412,7 @@ const mergeStep = (keyOrder) => (sources) => Effect.gen(function* () {
 *
 * @experimental
 */
-const merge = (streams) => {
+const merge = (streams, options) => {
 	const head = Array.headNonEmpty(streams);
 	const incompatible = Array.findFirst(Array.tailNonEmpty(streams), (stream) => stream.orderDirection !== head.orderDirection || !Equivalence(stream.keyLayout, head.keyLayout));
 	if (Option.isSome(incompatible)) throw new IncompatibleStreamsError({
@@ -355,17 +421,17 @@ const merge = (streams) => {
 		expectedKeyLayout: head.keyLayout,
 		actualKeyLayout: incompatible.value.keyLayout
 	});
-	return mergeUnchecked(streams);
+	return mergeUnchecked(streams, options);
 };
 /**
 * `merge` without the compatibility check—for re-merging branches that were
 * validated when the merge was built (narrowing never changes a branch's order
 * or key layout).
 */
-const mergeUnchecked = (streams) => {
+const mergeUnchecked = (streams, options) => {
 	const head = Array.headNonEmpty(streams);
-	const annotated = Stream.unwrap(Effect.map(Effect.forEach(streams, (stream) => Stream.toPull(stream.annotated)), (pulls) => Stream.unfold(Array.map(pulls, (pull) => MergeSource.NeedsPull({ pull })), mergeStep(Order$1(head.orderDirection)))));
-	return new QueryStream(head.orderDirection, head.keyLayout, annotated, void 0, (parsedBounds) => mergeUnchecked([narrowByParsedBounds(head, parsedBounds), ...Array.map(Array.tailNonEmpty(streams), (stream) => narrowByParsedBounds(stream, parsedBounds))]), () => mergeUnchecked([reverse(head), ...Array.map(Array.tailNonEmpty(streams), (stream) => reverse(stream))]));
+	const annotated = Stream.unwrap(Effect.map(Effect.forEach(streams, (stream) => Stream.toPull(stream.annotated)), (pulls) => Stream.unfold(Array.map(pulls, (pull) => MergeSource.NeedsPull({ pull })), mergeStep(Order$1(head.orderDirection), options?.dedupe === true))));
+	return new QueryStream(head.orderDirection, head.keyLayout, annotated, void 0, (parsedBounds) => mergeUnchecked([narrowByParsedBounds(head, parsedBounds), ...Array.map(Array.tailNonEmpty(streams), (stream) => narrowByParsedBounds(stream, parsedBounds))], options), () => mergeUnchecked([reverse(head), ...Array.map(Array.tailNonEmpty(streams), (stream) => reverse(stream))], options));
 };
 /**
 * Derive a stream by transforming each present document into `Some` (keep,
@@ -782,10 +848,10 @@ const narrowByParsedBounds = (self, parsedBounds) => {
 * The fallback for streams that don't know how to rebuild themselves.
 */
 const keyPredicates = (parsedBounds) => {
-	const complete$1 = (keyValues) => Result.getOrThrowWith(complete(parsedBounds.keyLayout, keyValues), identity);
+	const complete$2 = (keyValues) => Result.getOrThrowWith(complete(parsedBounds.keyLayout, keyValues), identity);
 	return {
-		aboveLower: (keyValues) => Result.getOrThrowWith(admittedByLower(parsedBounds)(complete$1(keyValues)), identity),
-		belowUpper: (keyValues) => Result.getOrThrowWith(admittedByUpper(parsedBounds)(complete$1(keyValues)), identity)
+		aboveLower: (keyValues) => Result.getOrThrowWith(admittedByLower(parsedBounds)(complete$2(keyValues)), identity),
+		belowUpper: (keyValues) => Result.getOrThrowWith(admittedByUpper(parsedBounds)(complete$2(keyValues)), identity)
 	};
 };
 const narrowInMemory = (self, parsedBounds) => {
@@ -795,6 +861,30 @@ const narrowInMemory = (self, parsedBounds) => {
 	return new QueryStream(self.orderDirection, self.keyLayout, pipe(self.annotated, dropOutOfRange, takeInRange), void 0, void 0, self.reverseWith === void 0 ? void 0 : () => narrowInMemory(reverse(self), parsedBounds));
 };
 /**
+* Collect the whole query stream or fail with `ReadBudgetExceededError` when a
+* read budget stops it. No partial array reaches the caller. Arbitrary I/O in
+* callbacks is not counted. Use `paginate` for data that can grow without
+* bound.
+*
+* With explicit options, each run gets a fresh budget. Without options, use the
+* enclosing budget if present, or the default row budget.
+*
+* @experimental
+*/
+const collect = dual((args) => isQueryStream(args[0]), Effect.fn("QueryStream.collect")(function* (self, options) {
+	const makeBudget = Schema.decodeEffect(Limits)({
+		maximumRowsRead: DEFAULT_MAXIMUM_ROWS_READ,
+		...options
+	}).pipe(Effect.catchTag("SchemaError", (cause) => Effect.die(new InvalidReadLimitError({ cause }))), Effect.flatMap(make$1));
+	const budget = yield* options === void 0 ? Effect.serviceOption(QueryStreamReadBudget).pipe(Effect.flatMap(Option.match({
+		onNone: () => makeBudget,
+		onSome: Effect.succeed
+	}))) : makeBudget;
+	const rows = yield* Stream.runCollect(self.toStream()).pipe(Effect.provideService(QueryStreamReadBudget, budget));
+	if (yield* budget.isStopped) return yield* new ReadBudgetExceededError(yield* budget.getReadCounts);
+	return rows;
+}));
+/**
 * `unique` found at least two emitted values. Narrow the query if at most one
 * value was expected, or use another consumer to handle multiple results.
 *
@@ -846,9 +936,9 @@ const unique = Effect.fn("QueryStream.unique")((self) => self.pipe(Stream.take(2
 * query semantics change, even if an old cursor is still accepted.
 *
 * Cursors expose boundary field names and key values, including document IDs.
-* They are neither opaque nor signed: clients can read or craft them. Do not
-* publicly paginate over sensitive indexed fields unless pinned with `eq`.
-* Never construct cursors from documents or decode them into `narrow` bounds.
+* Without a `cursorEnvelope`, clients can read or craft them. Do not publicly
+* paginate over sensitive indexed fields unless pinned with `eq`. Never
+* construct cursors from documents or decode them into `narrow` bounds.
 *
 * Return the whole result to reactive clients. React's
 * `useStreamPaginatedQuery` handles pinning, splitting, and invalid-cursor
@@ -858,19 +948,28 @@ const unique = Effect.fn("QueryStream.unique")((self) => self.pipe(Stream.take(2
 * @experimental
 */
 const paginate = dual(2, Effect.fn("QueryStream.paginate")(function* (self, options) {
+	const seal = (cursor) => options.cursorEnvelope === void 0 ? Effect.succeed(cursor) : options.cursorEnvelope.seal(cursor);
+	const open = (cursor) => options.cursorEnvelope === void 0 ? Effect.succeed(cursor) : options.cursorEnvelope.open(cursor).pipe(Effect.flatMap(Option.match({
+		onNone: () => Effect.die(new ConvexError({ paginationError: "InvalidCursor" })),
+		onSome: Effect.succeed
+	})));
+	const opened = yield* Effect.all({
+		start: options.cursor === null ? Effect.succeed(null) : open(options.cursor),
+		end: options.endCursor == null ? Effect.succeed(null) : open(options.endCursor)
+	});
 	const cursorSchema = fromKeyLayout(self.keyLayout);
 	const encodeCursor = Schema.encodeEffect(cursorSchema);
 	const decodeCursor = Schema.decodeEffect(cursorSchema);
-	const complete$2 = (keyValues) => Result.getOrThrowWith(complete(self.keyLayout, keyValues), identity);
+	const complete$1 = (keyValues) => Result.getOrThrowWith(complete(self.keyLayout, keyValues), identity);
 	const decoded = yield* Effect.all({
-		start: Option.match(Option.fromNullOr(options.cursor), {
+		start: Option.match(Option.fromNullOr(opened.start), {
 			onNone: () => Effect.succeed(Start.Beginning()),
 			onSome: (cursor) => Effect.map(decodeCursor(cursor), (key) => Start.After({
 				cursor,
 				key
 			}))
 		}),
-		range: Option.match(Option.fromNullishOr(options.endCursor), {
+		range: Option.match(Option.fromNullishOr(opened.end), {
 			onNone: () => Effect.succeed(Range.Unpinned()),
 			onSome: (cursor) => cursor === "[]" ? Effect.succeed(Range.ThroughEnd()) : Effect.map(decodeCursor(cursor), (key) => Range.ThroughKey({ key }))
 		})
@@ -882,7 +981,7 @@ const paginate = dual(2, Effect.fn("QueryStream.paginate")(function* (self, opti
 	if (request._tag === "Unchanged") return {
 		page: [],
 		isDone: false,
-		continueCursor: request.cursor
+		continueCursor: yield* seal(request.cursor)
 	};
 	const start = Option.map(request.after, (key) => ({
 		key: toPrefix(key),
@@ -904,14 +1003,14 @@ const paginate = dual(2, Effect.fn("QueryStream.paginate")(function* (self, opti
 		upper: start
 	}), identity));
 	const readBudget = yield* QueryStreamReadBudget;
-	const collected = yield* Stream.run(narrowed.annotated, Sink.fold(() => initial(), (state) => state._tag === "Reading", (state, { doc, keyValues }) => Effect.map(readBudget.isExhausted, (hitLimit) => record(request, state, doc, complete$2(keyValues), hitLimit))));
+	const collected = yield* Stream.run(narrowed.annotated, Sink.fold(() => initial(), (state) => state._tag === "Reading", (state, { doc, keyValues }) => Effect.map(readBudget.isStopped, (hitLimit) => record(request, state, doc, complete$1(keyValues), hitLimit))));
 	const stopped = yield* readBudget.isStopped;
 	const outcome = finish(request, collected, stopped);
 	if (Result.isFailure(outcome)) return yield* new ReadBudgetExceededError(yield* readBudget.getReadCounts);
-	const encode = (key) => encodeCursor(key).pipe(Effect.catchTag("SchemaError", Effect.die));
+	const encode = (key) => encodeCursor(key).pipe(Effect.catchTag("SchemaError", Effect.die), Effect.flatMap(seal));
 	const encodeSplit = (result) => Effect.gen(function* () {
 		const continueCursor = yield* Match.value(result.continuation).pipe(Match.tagsExhaustive({
-			End: () => Effect.succeed("[]"),
+			End: () => seal("[]"),
 			Key: ({ key }) => encode(key)
 		}));
 		return {
@@ -923,11 +1022,11 @@ const paginate = dual(2, Effect.fn("QueryStream.paginate")(function* (self, opti
 		};
 	});
 	return yield* Match.value(outcome.success).pipe(Match.tagsExhaustive({
-		Done: ({ page }) => Effect.succeed({
+		Done: ({ page }) => Effect.map(seal("[]"), (continueCursor) => ({
 			page: Array.fromIterable(page),
 			isDone: true,
-			continueCursor: "[]"
-		}),
+			continueCursor
+		})),
 		Continue: ({ page, key }) => Effect.map(encode(key), (continueCursor) => ({
 			page: Array.fromIterable(page),
 			isDone: false,
@@ -936,8 +1035,11 @@ const paginate = dual(2, Effect.fn("QueryStream.paginate")(function* (self, opti
 		SplitRequired: encodeSplit,
 		SplitRecommended: encodeSplit
 	}));
-}, (effect, _self, options) => Effect.provideServiceEffect(effect, QueryStreamReadBudget, Schema.decodeEffect(Limits)(options).pipe(Effect.catchTag("SchemaError", (cause) => Effect.die(new InvalidReadLimitError({ cause }))), Effect.flatMap(make$1)))));
+}, (effect, _self, options) => Effect.provideServiceEffect(effect, QueryStreamReadBudget, Schema.decodeEffect(Limits)({
+	maximumRowsRead: DEFAULT_MAXIMUM_ROWS_READ,
+	...options
+}).pipe(Effect.catchTag("SchemaError", (cause) => Effect.die(new InvalidReadLimitError({ cause }))), Effect.flatMap(make$1)))));
 //#endregion
-export { Element, EmptyInitialPageError, IncompatibleStreamsError, InnerStreamLayoutMismatchError, InnerStreamOrderMismatchError, InvalidPageSizeError, InvalidReadLimitError, MissingReversalRecipeError, NotUniqueError, QueryStream, QueryStream_exports, ReadBudgetExceededError, TypeId, distinct, empty, filter, filterEffect, flatMap, fromReflection, isQueryStream, map, mapEffect, merge, narrow, paginate, renameKey, reverse, unique };
+export { DEFAULT_MAXIMUM_ROWS_READ, DeferredStreamMismatchError, Element, EmptyInitialPageError, IncompatibleStreamsError, InnerStreamLayoutMismatchError, InnerStreamOrderMismatchError, InvalidPageSizeError, InvalidReadLimitError, MissingReversalRecipeError, NotUniqueError, QueryStream, QueryStream_exports, ReadBudgetExceededError, TypeId, collect, defer, distinct, empty, filter, filterEffect, flatMap, fromReflection, isQueryStream, map, mapEffect, merge, narrow, paginate, renameKey, reverse, unique };
 
 //# sourceMappingURL=QueryStream.js.map