@arbidocs/sdk 0.3.44 → 0.3.46

package/dist/index.cjs

@@ -4709,13 +4709,11 @@ async function listDocuments(arbi) {
   return requireData(await arbi.fetch.GET("/v1/document/list"), "Failed to fetch documents");
 }
 async function* listPaginated(arbi, options = {}) {
-  const { pageSize = 5e3, order = "id_asc", fields, signal } = options;
-  let offset = 0;
-  let pagesFetched = 0;
-  while (!signal?.aborted && pagesFetched < MAX_PAGES) {
+  const { pageSize = 5e3, order = "id_asc", fields, signal, lookahead = 3 } = options;
+  const fetchPage = async (pageOffset) => {
     const query = {
       limit: pageSize,
-      offset,
+      offset: pageOffset,
       order
     };
     if (fields) query.fields = fields;
@@ -4726,16 +4724,40 @@ async function* listPaginated(arbi, options = {}) {
     if (error) {
       throw new Error(typeof error === "string" ? error : JSON.stringify(error));
     }
+    return data ?? [];
+  };
+  if (signal?.aborted) return;
+  const depth = Math.max(1, Math.min(lookahead, MAX_PAGES));
+  let issued = 0;
+  let pagesYielded = 0;
+  let nextOffsetToIssue = 0;
+  let done = false;
+  const queue = [];
+  const tryEnqueue = () => {
+    while (!done && queue.length < depth && issued < MAX_PAGES) {
+      const p2 = fetchPage(nextOffsetToIssue);
+      p2.catch(() => {
+      });
+      queue.push(p2);
+      nextOffsetToIssue += pageSize;
+      issued++;
+    }
+  };
+  tryEnqueue();
+  while (queue.length > 0 && !signal?.aborted) {
+    const page = await queue.shift();
     if (signal?.aborted) return;
-    const page = data ?? [];
+    const isShort = page.length < pageSize;
+    if (isShort) done = true;
+    tryEnqueue();
     if (page.length > 0) {
       yield page;
-      offset += page.length;
-      pagesFetched++;
+      pagesYielded++;
     }
-    if (page.length < pageSize) return;
+    if (isShort) return;
+    if (pagesYielded >= MAX_PAGES) break;
   }
-  if (pagesFetched >= MAX_PAGES) {
+  if (!signal?.aborted && !done && pagesYielded >= MAX_PAGES) {
     console.warn(
       `[arbi-sdk] listPaginated hit MAX_PAGES (${MAX_PAGES}) \u2014 stopped. Workspace may have more than ${MAX_PAGES * pageSize} documents.`
     );