@gaganref/confect-server 10.3.1-fork.30 → 10.3.1-fork.32

package/src/QueryStream.ts

@@ -834,6 +834,21 @@ const fillMergeSource = <Doc, E>(
     BudgetLimited: () => Effect.succeed(source),
   });
 
+const consumeMergeHead = <Doc, E>(
+  source: MergeSource<Doc, E>,
+): MergeSource<Doc, E> =>
+  source._tag === "Buffered"
+    ? Option.match(Chunk.head(source.tail), {
+        onNone: () => MergeSource.NeedsPull({ pull: source.pull }),
+        onSome: (head) =>
+          MergeSource.Buffered({
+            pull: source.pull,
+            head,
+            tail: Chunk.drop(source.tail, 1),
+          }),
+      })
+    : source;
+
 /**
  * One step of the k-way merge as a pure unfold: fill every source, emit the
  * earliest head (ties go to the earliest source, keeping the merge stable), and
@@ -841,7 +856,10 @@ const fillMergeSource = <Doc, E>(
  * exhausted or an input stopped before its next key was known.
  */
 const mergeStep =
-  <Doc, E>(keyOrder: Order.Order<QueryStreamKeyValues.QueryStreamKeyValues>) =>
+  <Doc, E>(
+    keyOrder: Order.Order<QueryStreamKeyValues.QueryStreamKeyValues>,
+    dedupe: boolean,
+  ) =>
   (
     sources: ReadonlyArray<MergeSource<Doc, E>>,
   ): Effect.Effect<
@@ -883,36 +901,73 @@ const mergeStep =
           ),
       );
 
-      return Option.getOrUndefined(
-        Option.map(earliest, ({ index, element }) =>
-          Tuple.make(
-            element,
-            Array.map(filled, (source, sourceIndex) =>
-              sourceIndex === index && source._tag === "Buffered"
-                ? Option.match(Chunk.head(source.tail), {
-                    onNone: () => MergeSource.NeedsPull({ pull: source.pull }),
-                    onSome: (head) =>
-                      MergeSource.Buffered({
-                        pull: source.pull,
-                        head,
-                        tail: Chunk.drop(source.tail, 1),
-                      }),
-                  })
-                : source,
-            ),
+      if (Option.isNone(earliest)) return undefined;
+      if (!dedupe) {
+        const { index, element } = earliest.value;
+        return Tuple.make(
+          element,
+          Array.map(filled, (source, sourceIndex) =>
+            sourceIndex === index ? consumeMergeHead(source) : source,
           ),
-        ),
-      );
+        );
+      }
+      const keyValues = earliest.value.element.keyValues;
+      const matches = (source: MergeSource<Doc, E>) =>
+        source._tag === "Buffered" &&
+        keyOrder(source.head.keyValues, keyValues) === 0;
+      let representative = earliest.value;
+      let remaining = filled;
+      // Resolve the entire equal-key group before publishing its boundary.
+      // A budget stop leaves it for the next page, including any visible copy.
+      for (;;) {
+        representative = Array.reduce(
+          remaining,
+          representative,
+          (best, source, index) =>
+            source._tag === "Buffered" &&
+            matches(source) &&
+            Option.isSome(source.head.doc) &&
+            (Option.isNone(best.element.doc) || index < best.index)
+              ? new MergeCandidate({ index, element: source.head })
+              : best,
+        );
+        remaining = yield* Effect.forEach(
+          Array.map(remaining, (source) =>
+            matches(source) ? consumeMergeHead(source) : source,
+          ),
+          fillMergeSource,
+          { concurrency: (yield* readBudget.isUnlimited) ? "unbounded" : 1 },
+        );
+        if (
+          remaining.some((source) => MergeSource.$is("BudgetLimited")(source))
+        )
+          return undefined;
+        if (!remaining.some(matches))
+          return Tuple.make(representative.element, remaining);
+      }
     });
 
+/**
+ * Optional merge behavior. With `dedupe`, equal complete stored keys, including
+ * ID tiebreakers, emit once. The first visible copy in source order wins; a
+ * filtered copy cannot suppress a visible one. Different keys remain distinct.
+ *
+ * @experimental
+ */
+export interface MergeOptions {
+  readonly dedupe?: boolean;
+}
+
 /**
  * Combine streams sharing a key layout into one stream in key order. Use this
  * to query several index ranges at once, such as notes by multiple roles.
  *
  * The merge emits the smallest next key when ascending and the largest when
- * descending. Unlike `Stream.merge`, it does not interleave by arrival time.
- * Overlapping inputs are not deduplicated: a document matched twice appears
- * twice. Supply at least one input; use `empty` for an empty dynamic list.
+ * descending. Unlike `Stream.merge`, it does not interleave by arrival time. By
+ * default, a document matched twice appears twice. Pass `{ dedupe: true }` to
+ * coalesce equal complete keys, including ID tiebreakers. Resolving a group can
+ * prefetch the next key from each input; all such reads count against the
+ * shared budget. Supply at least one input; use `empty` for an empty list.
  *
  * Inputs need compatible document types and identical key layouts and
  * directions. TypeScript rejects known mismatches; runtime mismatches throw
@@ -934,6 +989,7 @@ export const merge = <
       QueryStream<Doc, KeyLabels, NoInfer<OrderDirection>, E, R>
     >,
   ],
+  options?: MergeOptions,
 ): QueryStream<Doc, KeyLabels, OrderDirection, E, R> => {
   const head = Array.headNonEmpty(streams);
   const incompatible = Array.findFirst(
@@ -950,7 +1006,7 @@ export const merge = <
       actualKeyLayout: incompatible.value.keyLayout,
     });
   }
-  return mergeUnchecked(streams);
+  return mergeUnchecked(streams, options);
 };
 
 /**
@@ -969,6 +1025,7 @@ const mergeUnchecked = <
     QueryStream<Doc, KeyLabels, OrderDirection, E, R>,
     ...ReadonlyArray<QueryStream<Doc, KeyLabels, OrderDirection, E, R>>,
   ],
+  options?: MergeOptions,
 ): QueryStream<Doc, KeyLabels, OrderDirection, E, R> => {
   const head = Array.headNonEmpty(streams);
   const annotated: Stream.Stream<
@@ -981,7 +1038,10 @@ const mergeUnchecked = <
       (pulls) =>
         Stream.unfold(
           Array.map(pulls, (pull) => MergeSource.NeedsPull({ pull })),
-          mergeStep<Doc, E>(QueryStreamKeyValues.Order(head.orderDirection)),
+          mergeStep<Doc, E>(
+            QueryStreamKeyValues.Order(head.orderDirection),
+            options?.dedupe === true,
+          ),
         ),
     ),
   );
@@ -995,17 +1055,25 @@ const mergeUnchecked = <
     // key space, so each branch converts them to its own index-key
     // space itself.
     (parsedBounds) =>
-      mergeUnchecked([
-        narrowByParsedBounds(head, parsedBounds),
-        ...Array.map(Array.tailNonEmpty(streams), (stream) =>
-          narrowByParsedBounds(stream, parsedBounds),
-        ),
-      ]),
+      mergeUnchecked(
+        [
+          narrowByParsedBounds(head, parsedBounds),
+          ...Array.map(Array.tailNonEmpty(streams), (stream) =>
+            narrowByParsedBounds(stream, parsedBounds),
+          ),
+        ],
+        options,
+      ),
     () =>
-      mergeUnchecked([
-        reverse(head),
-        ...Array.map(Array.tailNonEmpty(streams), (stream) => reverse(stream)),
-      ]),
+      mergeUnchecked(
+        [
+          reverse(head),
+          ...Array.map(Array.tailNonEmpty(streams), (stream) =>
+            reverse(stream),
+          ),
+        ],
+        options,
+      ),
   );
 };
 
@@ -2373,6 +2441,27 @@ export const unique = Effect.fn("QueryStream.unique")(
 // Pagination
 // -----------------------------------------------------------------------------
 
+/**
+ * Caller-owned envelope for every pagination cursor, including end sentinels.
+ * `open` validates a received cursor and returns `None` on a mismatch; paginate
+ * then dies with `ConvexError({ paginationError: "InvalidCursor" })` before
+ * decoding or reading. `seal` wraps every outgoing continuation and split.
+ * `seal` must return the same string for equal inputs so cursor equality stays
+ * stable.
+ *
+ * Use this to bind cursors to query semantics or sign them. Both callbacks may
+ * use Effect services. Confect owns no keys, fingerprints, or encryption
+ * policy.
+ *
+ * @experimental
+ */
+export interface CursorEnvelope<R = never> {
+  readonly open: (
+    cursor: string,
+  ) => Effect.Effect<Option.Option<string>, never, R>;
+  readonly seal: (cursor: string) => Effect.Effect<string, never, R>;
+}
+
 /**
  * Convex pagination options interpreted over a query stream's stored keys. Pass
  * a paginated query handler's `paginationOpts` directly to `paginate`.
@@ -2389,6 +2478,9 @@ export const unique = Effect.fn("QueryStream.unique")(
  * - `maximumBytesRead`: optional budget charging estimated document size on every
  *   read, not Convex's exact billed bytes. The final document can take the
  *   total over the limit because its size is known only after reading it.
+ * - `cursorEnvelope`: optional caller-owned validation and sealing. Both input
+ *   cursors are opened before decoding, including an end sentinel. Every output
+ *   cursor is sealed. A refused envelope ends in `InvalidCursor`.
  *
  * At either read limit, including zero, one extra row may be read to
  * distinguish completion from an overrun. It is counted but never emitted. An
@@ -2399,7 +2491,9 @@ export const unique = Effect.fn("QueryStream.unique")(
  *
  * @experimental
  */
-export type PaginateOptions = ConvexPaginationOptions;
+export type PaginateOptions<R = never> = ConvexPaginationOptions & {
+  readonly cursorEnvelope?: CursorEnvelope<R>;
+};
 
 /**
  * One page in Convex's pagination result shape. Return the whole result to
@@ -2453,9 +2547,9 @@ export type PaginationResult<Doc> = ConvexPaginationResult<Doc>;
  * query semantics change, even if an old cursor is still accepted.
  *
  * Cursors expose boundary field names and key values, including document IDs.
- * They are neither opaque nor signed: clients can read or craft them. Do not
- * publicly paginate over sensitive indexed fields unless pinned with `eq`.
- * Never construct cursors from documents or decode them into `narrow` bounds.
+ * Without a `cursorEnvelope`, clients can read or craft them. Do not publicly
+ * paginate over sensitive indexed fields unless pinned with `eq`. Never
+ * construct cursors from documents or decode them into `narrow` bounds.
  *
  * Return the whole result to reactive clients. React's
  * `useStreamPaginatedQuery` handles pinning, splitting, and invalid-cursor
@@ -2465,8 +2559,8 @@ export type PaginationResult<Doc> = ConvexPaginationResult<Doc>;
  * @experimental
  */
 export const paginate: {
-  (
-    options: PaginateOptions,
+  <R2 = never>(
+    options: PaginateOptions<R2>,
   ): <Doc, KeyLabels extends QueryStreamKeyLabels.QueryStreamKeyLabels, E, R>(
     self: QueryStream<
       Doc,
@@ -2475,8 +2569,18 @@ export const paginate: {
       E,
       R
     >,
-  ) => Effect.Effect<PaginationResult<Doc>, E | ReadBudgetExceededError, R>;
-  <Doc, KeyLabels extends QueryStreamKeyLabels.QueryStreamKeyLabels, E, R>(
+  ) => Effect.Effect<
+    PaginationResult<Doc>,
+    E | ReadBudgetExceededError,
+    R | R2
+  >;
+  <
+    Doc,
+    KeyLabels extends QueryStreamKeyLabels.QueryStreamKeyLabels,
+    E,
+    R,
+    R2 = never,
+  >(
     self: QueryStream<
       Doc,
       KeyLabels,
@@ -2484,8 +2588,8 @@ export const paginate: {
       E,
       R
     >,
-    options: PaginateOptions,
-  ): Effect.Effect<PaginationResult<Doc>, E | ReadBudgetExceededError, R>;
+    options: PaginateOptions<R2>,
+  ): Effect.Effect<PaginationResult<Doc>, E | ReadBudgetExceededError, R | R2>;
 } = dual(
   2,
   Effect.fn("QueryStream.paginate")(
@@ -2496,10 +2600,37 @@ export const paginate: {
       R,
       OrderDirection extends
         QueryStreamOrderDirection.QueryStreamOrderDirection,
+      R2 = never,
     >(
       self: QueryStream<Doc, KeyLabels, OrderDirection, E, R>,
-      options: PaginateOptions,
+      options: PaginateOptions<R2>,
     ) {
+      const seal = (cursor: string): Effect.Effect<string, never, R2> =>
+        options.cursorEnvelope === undefined
+          ? Effect.succeed(cursor)
+          : options.cursorEnvelope.seal(cursor);
+      const open = (cursor: string): Effect.Effect<string, never, R2> =>
+        options.cursorEnvelope === undefined
+          ? Effect.succeed(cursor)
+          : options.cursorEnvelope.open(cursor).pipe(
+              Effect.flatMap(
+                Option.match({
+                  onNone: () =>
+                    Effect.die(
+                      new ConvexError({ paginationError: "InvalidCursor" }),
+                    ),
+                  onSome: Effect.succeed,
+                }),
+              ),
+            );
+      const opened = yield* Effect.all({
+        start:
+          options.cursor === null ? Effect.succeed(null) : open(options.cursor),
+        end:
+          options.endCursor == null
+            ? Effect.succeed(null)
+            : open(options.endCursor),
+      });
       const cursorSchema = QueryStreamCursor.fromKeyLayout(self.keyLayout);
       const encodeCursor = Schema.encodeEffect(cursorSchema);
       const decodeCursor = Schema.decodeEffect(cursorSchema);
@@ -2509,7 +2640,7 @@ export const paginate: {
           identity,
         );
       const decoded = yield* Effect.all({
-        start: Option.match(Option.fromNullOr(options.cursor), {
+        start: Option.match(Option.fromNullOr(opened.start), {
           onNone: () => Effect.succeed(QueryStreamPagination.Start.Beginning()),
           onSome: (cursor) =>
             Effect.map(decodeCursor(cursor), (key) =>
@@ -2519,7 +2650,7 @@ export const paginate: {
               }),
             ),
         }),
-        range: Option.match(Option.fromNullishOr(options.endCursor), {
+        range: Option.match(Option.fromNullishOr(opened.end), {
           onNone: () => Effect.succeed(QueryStreamPagination.Range.Unpinned()),
           onSome: (cursor) =>
             cursor === QueryStreamCursor.END_CURSOR
@@ -2550,7 +2681,7 @@ export const paginate: {
         return {
           page: [],
           isDone: false,
-          continueCursor: request.cursor,
+          continueCursor: yield* seal(request.cursor),
         } satisfies PaginationResult<Doc>;
       }
       const start = Option.map(request.after, (key) => ({
@@ -2603,7 +2734,10 @@ export const paginate: {
         return yield* new ReadBudgetExceededError(counts);
       }
       const encode = (key: QueryStreamKey.Complete) =>
-        encodeCursor(key).pipe(Effect.catchTag("SchemaError", Effect.die));
+        encodeCursor(key).pipe(
+          Effect.catchTag("SchemaError", Effect.die),
+          Effect.flatMap(seal),
+        );
       const encodeSplit = (
         result: Extract<
           QueryStreamPagination.Outcome<Doc>,
@@ -2613,7 +2747,7 @@ export const paginate: {
         Effect.gen(function* () {
           const continueCursor = yield* Match.value(result.continuation).pipe(
             Match.tagsExhaustive({
-              End: () => Effect.succeed(QueryStreamCursor.END_CURSOR),
+              End: () => seal(QueryStreamCursor.END_CURSOR),
               Key: ({ key }) => encode(key),
             }),
           );
@@ -2628,11 +2762,15 @@ export const paginate: {
       return yield* Match.value(outcome.success).pipe(
         Match.tagsExhaustive({
           Done: ({ page }) =>
-            Effect.succeed({
-              page: Array.fromIterable(page),
-              isDone: true,
-              continueCursor: QueryStreamCursor.END_CURSOR,
-            } satisfies PaginationResult<Doc>),
+            Effect.map(
+              seal(QueryStreamCursor.END_CURSOR),
+              (continueCursor) =>
+                ({
+                  page: Array.fromIterable(page),
+                  isDone: true,
+                  continueCursor,
+                }) satisfies PaginationResult<Doc>,
+            ),
           Continue: ({ page, key }) =>
             Effect.map(
               encode(key),