convex 1.27.2 → 1.27.4-alpha.0

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 1.27.3
+
+- Convex CLI now respects `HTTPS_PROXY` / `HTTP_PROXY` environment variables.
+  This is generally useful but was motivated by making Convex run in remote
+  asynchronous agent environemnts like Codex.
+- Cutoffs for logging large and slow Transition messages have been lowered.
+
 ## 1.27.2
 
 - `npx convex run` sends function log output to stderr instead of stdout.

package/dist/browser.bundle.js

@@ -28,7 +28,7 @@ var convex = (() => {
   });
 
   // src/index.ts
-  var version = "1.27.2";
+  var version = "1.27.4-alpha.0";
 
   // src/values/base64.ts
   var base64_exports = {};
@@ -613,6 +613,19 @@ var convex = (() => {
       args: convexToJson(args)
     });
   }
+  function serializePaginatedPathAndArgs(udfPath, args, options) {
+    const { initialNumItems, id } = options;
+    const result = JSON.stringify({
+      type: "paginated",
+      udfPath: canonicalizeUdfPath(udfPath),
+      args: convexToJson(args),
+      options: convexToJson({ initialNumItems, id })
+    });
+    return result;
+  }
+  function serializedQueryTokenIsPaginated(token) {
+    return JSON.parse(token).type === "paginated";
+  }
 
   // src/browser/sync/local_state.ts
   var LocalSyncState = class {
@@ -1295,10 +1308,17 @@ var convex = (() => {
       return localStore.modifiedQueries;
     }
     /**
+     * "Raw" with respect to errors vs values, but query results still have
+     * optimistic updates applied.
+     *
      * @internal
      */
     rawQueryResult(queryToken) {
-      return this.queryResults.get(queryToken);
+      const query = this.queryResults.get(queryToken);
+      if (query === void 0) {
+        return void 0;
+      }
+      return query.result;
     }
     queryResult(queryToken) {
       const query = this.queryResults.get(queryToken);
@@ -2024,21 +2044,17 @@ var convex = (() => {
       this._logVerbose(
         `received ${prettyMessageMB} transition in ${prettyTransitionTime} at ${prettyBytesPerSecond}`
       );
-      if (transitionTransitTime > 1e4 && messageLength > 1e7) {
+      if (messageLength > 2e7) {
         this.logger.log(
-          `received query results totalling more than 10MB (${prettyMessageMB}) which took more than 10s (${prettyTransitionTime}) to arrive`
-        );
-      } else if (messageLength > 2e7) {
-        this.logger.log(
-          `received query results totalling more that 20MB (${prettyMessageMB}) which will take a long time to download on slower connections`
+          `received query results totaling more that 20MB (${prettyMessageMB}) which will take a long time to download on slower connections`
         );
       } else if (transitionTransitTime > 2e4) {
         this.logger.log(
-          `received query results totalling ${prettyMessageMB} which took more than 20s to arrive (${prettyTransitionTime})`
+          `received query results totaling ${prettyMessageMB} which took more than 20s to arrive (${prettyTransitionTime})`
         );
       }
       if (this.debug) {
-        if (transitionTransitTime > 1e4 || messageLength > 1e7) {
+        if (transitionTransitTime > 2e3) {
           this.sendMessage({
             type: "Event",
             eventType: "ClientReceivedTransition",
@@ -2748,7 +2764,7 @@ var convex = (() => {
             token,
             modification: {
               kind: "Updated",
-              result: optimisticResult.result
+              result: optimisticResult
             }
           };
         }),
@@ -2861,7 +2877,7 @@ var convex = (() => {
       return this.optimisticQueryResults.queryResult(queryToken);
     }
     /**
-     * Whether local query result is available for a toke.
+     * Whether local query result is available for a token.
      *
      * This method does not throw if the result is an error.
      *
@@ -3162,11 +3178,374 @@ var convex = (() => {
     }
   };
 
+  // src/browser/sync/pagination.ts
+  function asPaginationResult(value) {
+    if (typeof value !== "object" || value === null || !Array.isArray(value.page) || typeof value.isDone !== "boolean" || typeof value.continueCursor !== "string") {
+      throw new Error(`Not a valid paginated query result: ${value?.toString()}`);
+    }
+    return value;
+  }
+
+  // src/browser/sync/paginated_query_client.ts
+  var PaginatedQueryClient = class {
+    constructor(client, onTransition) {
+      this.client = client;
+      this.onTransition = onTransition;
+      this.lastTransitionTs = Long.fromNumber(0);
+      this.client.addOnTransitionHandler(
+        (transition) => this.onBaseTransition(transition)
+      );
+    }
+    paginatedQuerySet = /* @__PURE__ */ new Map();
+    // hold onto a real Transition so we can construct synthetic ones with that timestamp
+    lastTransitionTs;
+    /**
+     * Subscribe to a paginated query.
+     *
+     * @param name - The name of the paginated query function
+     * @param args - Arguments for the query (excluding paginationOpts)
+     * @param options - Pagination options including initialNumItems
+     * @returns Object with paginatedQueryToken and unsubscribe function
+     */
+    subscribe(name, args, options) {
+      const canonicalizedUdfPath = canonicalizeUdfPath(name);
+      const token = serializePaginatedPathAndArgs(
+        canonicalizedUdfPath,
+        args,
+        options
+      );
+      const unsubscribe = () => this.removePaginatedQuerySubscriber(token);
+      const existingEntry = this.paginatedQuerySet.get(token);
+      if (existingEntry) {
+        existingEntry.numSubscribers += 1;
+        return {
+          paginatedQueryToken: token,
+          unsubscribe
+        };
+      }
+      this.paginatedQuerySet.set(token, {
+        token,
+        canonicalizedUdfPath,
+        args,
+        numSubscribers: 1,
+        options: { initialNumItems: options.initialNumItems },
+        nextPageKey: 0,
+        pageKeys: [],
+        pageKeyToQuery: /* @__PURE__ */ new Map(),
+        ongoingSplits: /* @__PURE__ */ new Map(),
+        skip: false,
+        id: options.id
+      });
+      this.addPageToPaginatedQuery(token, null, options.initialNumItems);
+      return {
+        paginatedQueryToken: token,
+        unsubscribe
+      };
+    }
+    /**
+     * Get current results for a paginated query based on local state.
+     *
+     * Throws an error when one of the pages has errored.
+     */
+    localPaginatedQueryResult(name, args, options) {
+      const canonicalizedUdfPath = canonicalizeUdfPath(name);
+      const token = serializePaginatedPathAndArgs(
+        canonicalizedUdfPath,
+        args,
+        options
+      );
+      return this.localPaginatedQueryResultByToken(token);
+    }
+    /**
+     * @internal
+     */
+    localPaginatedQueryResultByToken(token) {
+      const paginatedQuery = this.paginatedQuerySet.get(token);
+      if (!paginatedQuery) {
+        return void 0;
+      }
+      const activePages = this.activePageQueryTokens(paginatedQuery);
+      if (activePages.length === 0) {
+        return {
+          results: [],
+          status: "LoadingFirstPage",
+          loadMore: (numItems) => {
+            return this.loadMoreOfPaginatedQuery(token, numItems);
+          }
+        };
+      }
+      let allResults = [];
+      let hasUndefined = false;
+      let isDone = false;
+      for (const pageToken of activePages) {
+        const result = this.client.localQueryResultByToken(pageToken);
+        if (result === void 0) {
+          hasUndefined = true;
+          isDone = false;
+          continue;
+        }
+        const paginationResult = asPaginationResult(result);
+        allResults = allResults.concat(paginationResult.page);
+        isDone = !!paginationResult.isDone;
+      }
+      let status;
+      if (hasUndefined) {
+        status = allResults.length === 0 ? "LoadingFirstPage" : "LoadingMore";
+      } else if (isDone) {
+        status = "Exhausted";
+      } else {
+        status = "CanLoadMore";
+      }
+      return {
+        results: allResults,
+        status,
+        loadMore: (numItems) => {
+          return this.loadMoreOfPaginatedQuery(token, numItems);
+        }
+      };
+    }
+    onBaseTransition(transition) {
+      const changedBaseTokens = transition.queries.map((q) => q.token);
+      const changed = this.queriesContainingTokens(changedBaseTokens);
+      let paginatedQueries = [];
+      if (changed.length > 0) {
+        this.processPaginatedQuerySplits(
+          changed,
+          (token) => this.client.localQueryResultByToken(token)
+        );
+        paginatedQueries = changed.map((token) => ({
+          token,
+          modification: {
+            kind: "Updated",
+            result: this.localPaginatedQueryResultByToken(token)
+          }
+        }));
+      }
+      const extendedTransition = {
+        ...transition,
+        paginatedQueries
+      };
+      this.onTransition(extendedTransition);
+    }
+    /**
+     * Load more items for a paginated query.
+     *
+     * This *always* causes a transition, the status of the query
+     * has probably changed from "CanLoadMore" to "LoadingMore".
+     * Data might have changed too: maybe a subscription to this page
+     * query already exists (unlikely but possible) or this page query
+     * has an optimistic update providing some initial data.
+     *
+     * @internal
+     */
+    loadMoreOfPaginatedQuery(token, numItems) {
+      this.mustGetPaginatedQuery(token);
+      const lastPageToken = this.queryTokenForLastPageOfPaginatedQuery(token);
+      const lastPageResult = this.client.localQueryResultByToken(lastPageToken);
+      if (!lastPageResult) {
+        return false;
+      }
+      const paginationResult = asPaginationResult(lastPageResult);
+      if (paginationResult.isDone) {
+        return false;
+      }
+      this.addPageToPaginatedQuery(
+        token,
+        paginationResult.continueCursor,
+        numItems
+      );
+      const loadMoreTransition = {
+        timestamp: this.lastTransitionTs,
+        reflectedMutations: [],
+        queries: [],
+        paginatedQueries: [
+          {
+            token,
+            modification: {
+              kind: "Updated",
+              result: this.localPaginatedQueryResultByToken(token)
+            }
+          }
+        ]
+      };
+      this.onTransition(loadMoreTransition);
+      return true;
+    }
+    /**
+     * @internal
+     */
+    queriesContainingTokens(queryTokens) {
+      if (queryTokens.length === 0) {
+        return [];
+      }
+      const changed = [];
+      const queryTokenSet = new Set(queryTokens);
+      for (const [paginatedToken, paginatedQuery] of this.paginatedQuerySet) {
+        for (const pageToken of this.allQueryTokens(paginatedQuery)) {
+          if (queryTokenSet.has(pageToken)) {
+            changed.push(paginatedToken);
+            break;
+          }
+        }
+      }
+      return changed;
+    }
+    /**
+     * @internal
+     */
+    processPaginatedQuerySplits(changed, getResult) {
+      for (const paginatedQueryToken of changed) {
+        const paginatedQuery = this.mustGetPaginatedQuery(paginatedQueryToken);
+        const { ongoingSplits, pageKeyToQuery, pageKeys } = paginatedQuery;
+        for (const [pageKey, [splitKey1, splitKey2]] of ongoingSplits) {
+          const bothNewPagesLoaded = getResult(pageKeyToQuery.get(splitKey1).queryToken) !== void 0 && getResult(pageKeyToQuery.get(splitKey2).queryToken) !== void 0;
+          if (bothNewPagesLoaded) {
+            this.completePaginatedQuerySplit(
+              paginatedQuery,
+              pageKey,
+              splitKey1,
+              splitKey2
+            );
+          }
+        }
+        for (const pageKey of pageKeys) {
+          if (ongoingSplits.has(pageKey)) {
+            continue;
+          }
+          const pageToken = pageKeyToQuery.get(pageKey).queryToken;
+          const pageResult = getResult(pageToken);
+          if (!pageResult) {
+            continue;
+          }
+          const result = asPaginationResult(pageResult);
+          const shouldSplit = result.splitCursor && (result.pageStatus === "SplitRecommended" || result.pageStatus === "SplitRequired" || // This client-driven page splitting condition will change in the future.
+          result.page.length > paginatedQuery.options.initialNumItems * 2);
+          if (shouldSplit) {
+            this.splitPaginatedQueryPage(
+              paginatedQuery,
+              pageKey,
+              result.splitCursor,
+              // we just checked
+              result.continueCursor
+            );
+          }
+        }
+      }
+    }
+    splitPaginatedQueryPage(paginatedQuery, pageKey, splitCursor, continueCursor) {
+      const splitKey1 = paginatedQuery.nextPageKey++;
+      const splitKey2 = paginatedQuery.nextPageKey++;
+      const paginationOpts = {
+        cursor: continueCursor,
+        numItems: paginatedQuery.options.initialNumItems,
+        id: paginatedQuery.id
+      };
+      const firstSubscription = this.client.subscribe(
+        paginatedQuery.canonicalizedUdfPath,
+        {
+          ...paginatedQuery.args,
+          paginationOpts: {
+            ...paginationOpts,
+            cursor: null,
+            // Start from beginning for first split
+            endCursor: splitCursor
+          }
+        }
+      );
+      paginatedQuery.pageKeyToQuery.set(splitKey1, firstSubscription);
+      const secondSubscription = this.client.subscribe(
+        paginatedQuery.canonicalizedUdfPath,
+        {
+          ...paginatedQuery.args,
+          paginationOpts: {
+            ...paginationOpts,
+            cursor: splitCursor,
+            endCursor: continueCursor
+          }
+        }
+      );
+      paginatedQuery.pageKeyToQuery.set(splitKey2, secondSubscription);
+      paginatedQuery.ongoingSplits.set(pageKey, [splitKey1, splitKey2]);
+    }
+    /**
+     * @internal
+     */
+    addPageToPaginatedQuery(token, continueCursor, numItems) {
+      const paginatedQuery = this.mustGetPaginatedQuery(token);
+      const pageKey = paginatedQuery.nextPageKey++;
+      const paginationOpts = {
+        cursor: continueCursor,
+        numItems,
+        id: paginatedQuery.id
+      };
+      const pageArgs = {
+        ...paginatedQuery.args,
+        paginationOpts
+      };
+      const subscription = this.client.subscribe(
+        paginatedQuery.canonicalizedUdfPath,
+        pageArgs
+      );
+      paginatedQuery.pageKeys.push(pageKey);
+      paginatedQuery.pageKeyToQuery.set(pageKey, subscription);
+      return subscription;
+    }
+    removePaginatedQuerySubscriber(token) {
+      const paginatedQuery = this.paginatedQuerySet.get(token);
+      if (!paginatedQuery) {
+        return;
+      }
+      paginatedQuery.numSubscribers -= 1;
+      if (paginatedQuery.numSubscribers > 0) {
+        return;
+      }
+      for (const subscription of paginatedQuery.pageKeyToQuery.values()) {
+        subscription.unsubscribe();
+      }
+      this.paginatedQuerySet.delete(token);
+    }
+    completePaginatedQuerySplit(paginatedQuery, pageKey, splitKey1, splitKey2) {
+      const originalQuery = paginatedQuery.pageKeyToQuery.get(pageKey);
+      paginatedQuery.pageKeyToQuery.delete(pageKey);
+      const pageIndex = paginatedQuery.pageKeys.indexOf(pageKey);
+      paginatedQuery.pageKeys.splice(pageIndex, 1, splitKey1, splitKey2);
+      paginatedQuery.ongoingSplits.delete(pageKey);
+      originalQuery.unsubscribe();
+    }
+    /** The query tokens for all active pages, in result order */
+    activePageQueryTokens(paginatedQuery) {
+      return paginatedQuery.pageKeys.map(
+        (pageKey) => paginatedQuery.pageKeyToQuery.get(pageKey).queryToken
+      );
+    }
+    allQueryTokens(paginatedQuery) {
+      return Array.from(paginatedQuery.pageKeyToQuery.values()).map(
+        (sub) => sub.queryToken
+      );
+    }
+    queryTokenForLastPageOfPaginatedQuery(token) {
+      const paginatedQuery = this.mustGetPaginatedQuery(token);
+      const lastPageKey = paginatedQuery.pageKeys[paginatedQuery.pageKeys.length - 1];
+      if (lastPageKey === void 0) {
+        throw new Error(`No pages for paginated query ${token}`);
+      }
+      return paginatedQuery.pageKeyToQuery.get(lastPageKey).queryToken;
+    }
+    mustGetPaginatedQuery(token) {
+      const paginatedQuery = this.paginatedQuerySet.get(token);
+      if (!paginatedQuery) {
+        throw new Error("paginated query no longer exists for token " + token);
+      }
+      return paginatedQuery;
+    }
+  };
+
   // src/browser/simple_client.ts
   var defaultWebSocketConstructor;
   var ConvexClient = class {
     listeners;
     _client;
+    _paginatedClient;
     // A synthetic server event to run callbacks the first time
     callNewListenersWithCurrentValuesTimer;
     _closed;
@@ -3181,6 +3560,10 @@ var convex = (() => {
       if (this._client) return this._client;
       throw new Error("ConvexClient is disabled");
     }
+    get paginatedClient() {
+      if (this._paginatedClient) return this._paginatedClient;
+      throw new Error("ConvexClient is disabled");
+    }
     get disabled() {
       return this._disabled;
     }
@@ -3205,9 +3588,15 @@ var convex = (() => {
       if (!this.disabled) {
         this._client = new BaseConvexClient(
           address,
-          (updatedQueries) => this._transition(updatedQueries),
+          () => {
+          },
+          // NOP, let the paginated query client do it all
           baseOptions
         );
+        this._paginatedClient = new PaginatedQueryClient(
+          this._client,
+          (transition) => this._transition(transition)
+        );
       }
       this.listeners = /* @__PURE__ */ new Set();
     }
@@ -3245,15 +3634,7 @@ var convex = (() => {
      */
     onUpdate(query, args, callback, onError) {
       if (this.disabled) {
-        const disabledUnsubscribe = () => {
-        };
-        const unsubscribeProps2 = {
-          unsubscribe: disabledUnsubscribe,
-          getCurrentValue: () => void 0,
-          getQueryLogs: () => void 0
-        };
-        Object.assign(disabledUnsubscribe, unsubscribeProps2);
-        return disabledUnsubscribe;
+        return this.createDisabledUnsubscribe();
       }
       const { queryToken, unsubscribe } = this.client.subscribe(
         getFunctionName(query),
@@ -3266,7 +3647,8 @@ var convex = (() => {
         unsubscribe,
         hasEverRun: false,
         query,
-        args
+        args,
+        paginationOptions: void 0
       };
       this.listeners.add(queryInfo);
       if (this.queryResultReady(queryToken) && this.callNewListenersWithCurrentValuesTimer === void 0) {
@@ -3290,19 +3672,100 @@ var convex = (() => {
       Object.assign(ret, unsubscribeProps);
       return ret;
     }
+    /**
+     * Call a callback whenever a new result for a paginated query is received.
+     *
+     * @param query - A {@link server.FunctionReference} for the public query to run.
+     * @param args - The arguments to run the query with.
+     * @param options - Options for the paginated query including initialNumItems and id.
+     * @param callback - Function to call when the query result updates.
+     * @param onError - Function to call when the query result updates with an error.
+     *
+     * @return an {@link Unsubscribe} function to stop calling the callback.
+     */
+    onPaginatedUpdate(query, args, options, callback, onError) {
+      if (this.disabled) {
+        return this.createDisabledUnsubscribe();
+      }
+      const paginationOptions = {
+        initialNumItems: options.initialNumItems,
+        id: -1
+      };
+      const { paginatedQueryToken, unsubscribe } = this.paginatedClient.subscribe(
+        getFunctionName(query),
+        args,
+        // Simple client doesn't use IDs, there's no expectation that these queries remain separate.
+        paginationOptions
+      );
+      const queryInfo = {
+        queryToken: paginatedQueryToken,
+        callback,
+        onError,
+        unsubscribe,
+        hasEverRun: false,
+        query,
+        args,
+        paginationOptions
+      };
+      this.listeners.add(queryInfo);
+      if (!!this.paginatedClient.localPaginatedQueryResultByToken(
+        paginatedQueryToken
+      ) && this.callNewListenersWithCurrentValuesTimer === void 0) {
+        this.callNewListenersWithCurrentValuesTimer = setTimeout(
+          () => this.callNewListenersWithCurrentValues(),
+          0
+        );
+      }
+      const unsubscribeProps = {
+        unsubscribe: () => {
+          if (this.closed) {
+            return;
+          }
+          this.listeners.delete(queryInfo);
+          unsubscribe();
+        },
+        getCurrentValue: () => {
+          const result = this.paginatedClient.localPaginatedQueryResult(
+            getFunctionName(query),
+            args,
+            paginationOptions
+          );
+          return result;
+        },
+        getQueryLogs: () => []
+        // Paginated queries don't aggregate their logs
+      };
+      const ret = unsubscribeProps.unsubscribe;
+      Object.assign(ret, unsubscribeProps);
+      return ret;
+    }
     // Run all callbacks that have never been run before if they have a query
     // result available now.
     callNewListenersWithCurrentValues() {
       this.callNewListenersWithCurrentValuesTimer = void 0;
-      this._transition([], true);
+      this._transition({ queries: [], paginatedQueries: [] }, true);
     }
     queryResultReady(queryToken) {
       return this.client.hasLocalQueryResultByToken(queryToken);
     }
+    createDisabledUnsubscribe() {
+      const disabledUnsubscribe = () => {
+      };
+      const unsubscribeProps = {
+        unsubscribe: disabledUnsubscribe,
+        getCurrentValue: () => void 0,
+        getQueryLogs: () => void 0
+      };
+      Object.assign(disabledUnsubscribe, unsubscribeProps);
+      return disabledUnsubscribe;
+    }
     async close() {
       if (this.disabled) return;
       this.listeners.clear();
       this._closed = true;
+      if (this._paginatedClient) {
+        this._paginatedClient = void 0;
+      }
       return this.client.close();
     }
     /**
@@ -3334,14 +3797,27 @@ var convex = (() => {
     /**
      * @internal
      */
-    _transition(updatedQueries, callNewListeners = false) {
+    _transition({
+      queries,
+      paginatedQueries
+    }, callNewListeners = false) {
+      const updatedQueries = [
+        ...queries.map((q) => q.token),
+        ...paginatedQueries.map((q) => q.token)
+      ];
       for (const queryInfo of this.listeners) {
         const { callback, queryToken, onError, hasEverRun } = queryInfo;
-        if (updatedQueries.includes(queryToken) || callNewListeners && !hasEverRun && this.client.hasLocalQueryResultByToken(queryToken)) {
+        const isPaginatedQuery = serializedQueryTokenIsPaginated(queryToken);
+        const hasResultReady = isPaginatedQuery ? !!this.paginatedClient.localPaginatedQueryResultByToken(queryToken) : this.client.hasLocalQueryResultByToken(queryToken);
+        if (updatedQueries.includes(queryToken) || callNewListeners && !hasEverRun && hasResultReady) {
           queryInfo.hasEverRun = true;
           let newValue;
           try {
-            newValue = this.client.localQueryResultByToken(queryToken);
+            if (isPaginatedQuery) {
+              newValue = this.paginatedClient.localPaginatedQueryResultByToken(queryToken);
+            } else {
+              newValue = this.client.localQueryResultByToken(queryToken);
+            }
           } catch (error) {
             if (!(error instanceof Error)) throw error;
             if (onError) {