@pkcprotocol/pkc-js 0.0.107 → 0.0.108

package/dist/browser/runtime/node/community/page-generator.js

@@ -1,18 +1,193 @@
-import { calculateStringSizeSameAsIpfsAddCidV0, hideClassPrivateProps, retryKuboIpfsAddAndProvide, timestamp } from "../../../util.js";
+import { calculateUnixFsDagSizeCidV0, hideClassPrivateProps, retryKuboIpfsAddAndProvide, timestamp } from "../../../util.js";
+import v8 from "node:v8";
+import { Buffer } from "buffer";
 import assert from "assert";
-import { keys, omit } from "remeda";
 import { stringify as deterministicStringify } from "safe-stable-stringify";
 import env from "../../../version.js";
-import { POSTS_SORT_TYPES, POST_REPLIES_SORT_TYPES, TIMEFRAMES_TO_SECONDS, REPLY_REPLIES_SORT_TYPES } from "../../../pages/util.js";
 import { PKCError } from "../../../pkc-error.js";
+import { applyPageSortExclusions, createDescendantsLookup, orderPageCommentsByScore, partitionPageCommentsForSort, scorePageCommentsWithFile, stripRepliesFromPageComment } from "../../../pages/page-sort-scoring.js";
 import Logger from "../../../logger.js";
 import { cleanUpBeforePublishing, signCommentUpdateForChallengeVerification } from "../../../signer/signatures.js";
 import { deriveCommentIpfsFromCommentTableRow } from "../util.js";
 import { sha256 } from "js-sha256";
-async function getSerializedCommentsSize(comments, hasNextCid) {
-    const payload = hasNextCid ? { comments, nextCid: "QmXsYKgNH7XoZXdLko5uDvtWSRNE2AXuQ4u8KxVpCacrZx" } : { comments };
-    const serializedPayload = JSON.stringify(payload);
-    return await calculateStringSizeSameAsIpfsAddCidV0(serializedPayload);
+// The wire shape (`{ pages, pageCids? }`) of either generation result. `pages` holds only the preloaded sorts'
+// embedded first pages; a preloaded sort that overflowed into more chunks continues through pages[sort].nextCid,
+// and a sort that is not embedded is reachable through pageCids only.
+export function wirePagesFromGeneration(generated) {
+    if ("singlePreloadedPage" in generated)
+        return { pages: generated.singlePreloadedPage };
+    return { pages: generated.pages, ...(generated.pageCids ? { pageCids: generated.pageCids } : {}) };
+}
+const MB = 1024 * 1024;
+const NON_PRELOADED_FIRST_PAGE_SIZE = MB; // pageCids first pages are always capped at 1mib, regardless of the preload budget
+const SAFETY_MARGIN = 1024; // 1KiB under every page limit when chunking
+const NEXT_CID_PLACEHOLDER = "QmXsYKgNH7XoZXdLko5uDvtWSRNE2AXuQ4u8KxVpCacrZx"; // any CIDv0: what a nextCid costs
+// A page is its entries' canonical JSON joined inside this wrapper: keys sorted, and "comments" sorts before
+// "nextCid", so the join is byte-identical to safe-stable-stringify of the page object (issue #351).
+const PAGE_OPEN = '{"comments":[';
+const PAGE_CLOSE = "]}";
+const pageCloseWithNextCid = (nextCid) => `],"nextCid":${JSON.stringify(nextCid)}}`;
+const PAGE_OPEN_BYTES = Buffer.byteLength(PAGE_OPEN);
+const PAGE_CLOSE_BYTES = Buffer.byteLength(PAGE_CLOSE);
+const PAGE_CLOSE_WITH_CID_BYTES = Buffer.byteLength(pageCloseWithNextCid(NEXT_CID_PLACEHOLDER));
+// The UTF-8 size of a page holding entries of the given sizes
+function pageBytes(entrySizes, hasNextCid) {
+    let bytes = PAGE_OPEN_BYTES + (hasNextCid ? PAGE_CLOSE_WITH_CID_BYTES : PAGE_CLOSE_BYTES) + Math.max(entrySizes.length - 1, 0);
+    for (const size of entrySizes)
+        bytes += size;
+    return bytes;
+}
+// What `ipfs add` reports for a page of these entries
+const pageDagSize = (entrySizes, hasNextCid) => calculateUnixFsDagSizeCidV0(pageBytes(entrySizes, hasNextCid));
+const SERIALIZE_BATCH = 256; // entries resolved and serialized per database round trip
+// How many bytes of serialized entries a community keeps between generations: an eighth of the heap limit (so
+// --max-old-space-size scales it), never under 32 MiB or over 512 MiB. Past it, pages re-serialize what they need.
+function serializedEntryCacheBudgetBytes() {
+    const heapLimit = v8.getHeapStatistics().heap_size_limit;
+    return Math.min(Math.max(Math.floor(heapLimit / 8), 32 * MB), 512 * MB);
+}
+// Serialized entries kept across generations, by cid and shape, valid while the CommentUpdate's updatedAt is
+// unchanged: a CommentUpdate is recalculated (new updatedAt) whenever anything under the comment changes, so the
+// same (cid, updatedAt) serializes to the same bytes. The shape is whether the entry carries `replies`: one
+// generation loads the same reply twice when a post has nested and flat sorts, with its reply tree for the nested
+// sorts and without `replies` for the flat ones (queryFlattenedPageReplies excludes the column), and the two must
+// not share bytes, or a flat page would embed the nested JSON. Bounded by a byte budget; a changed entry replaces
+// its old bytes, a new entry is kept only while the budget allows, and entries a generation did not see are
+// dropped after it, so a steady board costs a serialization per changed post and the budget's worth of memory
+// (issue #351).
+class EntryJsonCache {
+    constructor(_budgetBytes) {
+        this._budgetBytes = _budgetBytes;
+        this._byKey = new Map();
+        this._bytes = 0;
+    }
+    static _key(entry) {
+        return entry.commentUpdate.replies === undefined ? entry.commentUpdate.cid : `${entry.commentUpdate.cid}+replies`;
+    }
+    get(entry) {
+        const cached = this._byKey.get(EntryJsonCache._key(entry));
+        return cached && cached.updatedAt === entry.commentUpdate.updatedAt ? cached : undefined;
+    }
+    offer(entry, json, bytes) {
+        const key = EntryJsonCache._key(entry);
+        const previous = this._byKey.get(key);
+        const budgetAfter = this._bytes - (previous?.bytes ?? 0) + bytes;
+        if (budgetAfter > this._budgetBytes) {
+            if (previous)
+                this._drop(key);
+            return;
+        }
+        this._byKey.set(key, { cid: entry.commentUpdate.cid, updatedAt: entry.commentUpdate.updatedAt, json, bytes });
+        this._bytes = budgetAfter;
+    }
+    // Keep only the cids a generation listed (a purged or excluded post frees its bytes)
+    retainOnly(cids) {
+        for (const [key, cached] of [...this._byKey.entries()])
+            if (!cids.has(cached.cid))
+                this._drop(key);
+    }
+    _drop(key) {
+        const cached = this._byKey.get(key);
+        if (!cached)
+            return;
+        this._byKey.delete(key);
+        this._bytes -= cached.bytes;
+    }
+}
+// Admission by bytes for the pages being built at once: sorts add their pages in parallel, but a page's string (its
+// entries fetched plus the join) lives in memory until its add returns, and the doubling page sizes make late pages
+// of a large board hundreds of MiB. A page waits until the bytes in flight fit the budget; one that exceeds the
+// budget alone runs when nothing else is in flight, so generation always makes progress (issue #351).
+class PageBytesBudget {
+    constructor(_budgetBytes) {
+        this._budgetBytes = _budgetBytes;
+        this._inFlight = 0;
+        this._waiting = [];
+    }
+    _fits(bytes) {
+        return this._inFlight === 0 || this._inFlight + bytes <= this._budgetBytes;
+    }
+    async acquire(bytes) {
+        if (this._fits(bytes))
+            this._inFlight += bytes;
+        else
+            await new Promise((resume) => this._waiting.push({ bytes, resume })); // counted by the release that admits it
+        let released = false;
+        return () => {
+            if (released)
+                return;
+            released = true;
+            this._inFlight -= bytes;
+            // Admit waiters in arrival order while they fit, counting each before it resumes so a caller that
+            // releases and acquires again in the same tick cannot slip in ahead of it
+            for (let i = 0; i < this._waiting.length;) {
+                if (!this._fits(this._waiting[i].bytes)) {
+                    i++;
+                    continue;
+                }
+                const [next] = this._waiting.splice(i, 1);
+                this._inFlight += next.bytes;
+                next.resume();
+            }
+        };
+    }
+}
+// The serialized form of one generation's entries: each entry's canonical page JSON and that JSON's UTF-8 size.
+// Sizes are kept for every entry, since every sort's chunking needs them; the strings live in the cache, so a
+// board's resolved reply trees are never all in memory. Sizing streams: entries the cache does not hold are
+// resolved and serialized a batch at a time; a page fetches what the cache does not hold in one batch when it is
+// built (issue #351).
+class SerializedEntries {
+    constructor(_source, _cache) {
+        this._source = _source;
+        this._cache = _cache;
+        this._sizes = new Map();
+        this.sizeOf = (entry) => {
+            const size = this._sizes.get(entry);
+            assert(size !== undefined, "entry was not serialized before chunking");
+            return size;
+        };
+    }
+    sizeAll(entries) {
+        const pending = [];
+        for (const entry of entries) {
+            if (this._sizes.has(entry))
+                continue;
+            const cached = this._cache.get(entry);
+            if (cached)
+                this._sizes.set(entry, cached.bytes);
+            else
+                pending.push(entry);
+        }
+        for (let start = 0; start < pending.length; start += SERIALIZE_BATCH) {
+            const batch = pending.slice(start, start + SERIALIZE_BATCH);
+            const strings = this._source.serialize(batch);
+            for (let i = 0; i < batch.length; i++) {
+                const bytes = Buffer.byteLength(strings[i]);
+                this._sizes.set(batch[i], bytes);
+                this._cache.offer(batch[i], strings[i], bytes);
+            }
+        }
+    }
+    // The JSON of each entry, in the given order
+    stringsOf(entries) {
+        const strings = new Array(entries.length);
+        const missing = [];
+        for (let i = 0; i < entries.length; i++) {
+            const cached = this._cache.get(entries[i]);
+            if (cached)
+                strings[i] = cached.json;
+            else
+                missing.push(i);
+        }
+        for (let start = 0; start < missing.length; start += SERIALIZE_BATCH) {
+            const indexes = missing.slice(start, start + SERIALIZE_BATCH);
+            const serialized = this._source.serialize(indexes.map((i) => entries[i]));
+            for (let k = 0; k < indexes.length; k++)
+                strings[indexes[k]] = serialized[k];
+        }
+        return strings;
+    }
 }
 export class PageGenerator {
     constructor(community) {
@@ -23,7 +198,7 @@ export class PageGenerator {
         const ipfsClient = this._community._clientsManager.getDefaultKuboRpcClient();
         const listOfPage = new Array(chunks.length);
         const cids = new Array(chunks.length);
-        let expectedSize = 1024 * 1024 * Math.pow(2, chunks.length - 1); // expected size of last page
+        let expectedSize = MB * Math.pow(2, chunks.length - 1); // expected size of last page
         for (let i = chunks.length - 1; i >= 0; i--) {
             const modQueuePageIpfs = { nextCid: cids[i + 1], comments: chunks[i] };
             if (!modQueuePageIpfs.nextCid)
@@ -50,134 +225,90 @@ export class PageGenerator {
         }
         return { pages: listOfPage, cids };
     }
-    async addCommentChunksToIpfs(chunks, sortName) {
+    // Add one sort's chunks, last page first so each page knows its nextCid. Every page is the join of its entries'
+    // serialized JSON, sized exactly before the add; a preloaded sort's first chunk is not added but returned as the
+    // page object the record embeds (its entries materialized from the database, nested replies resolved).
+    async _addChunksToIpfs({ chunks, sortName, serialized, embedFirst, source, budget }) {
         assert(chunks.length > 0);
-        const ipfsClient = this._community._clientsManager.getDefaultKuboRpcClient();
-        const listOfPage = new Array(chunks.length);
         const cids = new Array(chunks.length);
-        let curMaxPageSize = 1024 * 1024 * Math.pow(2, chunks.length - 1); // expected size of last page
-        for (let pageNum = chunks.length - 1; pageNum >= 0; pageNum--) {
-            const pageIpfs = { nextCid: cids[pageNum + 1], comments: chunks[pageNum] };
-            if (!pageIpfs.nextCid)
-                delete pageIpfs.nextCid; // we don't to include undefined anywhere in the protocol
-            const stringifiedPageIpfs = deterministicStringify(pageIpfs);
-            const calculatedSizeOfStringifedPageIpfs = await calculateStringSizeSameAsIpfsAddCidV0(stringifiedPageIpfs);
-            if (calculatedSizeOfStringifedPageIpfs > curMaxPageSize)
+        for (let pageNum = chunks.length - 1; pageNum >= (embedFirst ? 1 : 0); pageNum--) {
+            const nextCid = cids[pageNum + 1];
+            // pageCids pages may grow 1, 2, 4... MiB; the pages after an embedded first page start at 1 MiB
+            const maximumPageSize = MB * Math.pow(2, embedFirst ? Math.max(pageNum - 1, 0) : pageNum);
+            const entries = chunks[pageNum];
+            const entrySizes = entries.map(serialized.sizeOf);
+            const calculatedSize = pageDagSize(entrySizes, nextCid !== undefined);
+            if (calculatedSize > maximumPageSize)
                 throw new PKCError("ERR_PAGE_GENERATED_IS_OVER_EXPECTED_SIZE", {
-                    calculatedSizeOfStringifedPageIpfs,
-                    pageIpfs,
-                    expectedSize: curMaxPageSize,
+                    calculatedSizeOfStringifedPageIpfs: calculatedSize,
+                    expectedSize: maximumPageSize,
+                    commentCount: entries.length,
                     sortName,
                     pageNum
                 });
-            const addRes = await retryKuboIpfsAddAndProvide({
-                ipfsClient: ipfsClient._client,
-                log: Logger("pkc-js:page-generator:addCommentChunksToIpfs"),
-                content: stringifiedPageIpfs,
-                addOptions: { pin: true },
-                provideOptions: { recursive: true },
-                provideInBackground: true
-            });
-            if (addRes.size > curMaxPageSize)
-                throw new PKCError("ERR_PAGE_GENERATED_IS_OVER_EXPECTED_SIZE", {
-                    addRes,
-                    pageIpfs,
-                    expectedSize: curMaxPageSize,
-                    sortName,
-                    pageNum
-                });
-            cids[pageNum] = addRes.path;
-            listOfPage[pageNum] = pageIpfs;
-            curMaxPageSize = curMaxPageSize / 2; // we're going backward now
+            // the entries' strings plus their join are both held until the add returns
+            const release = await budget.acquire(2 * pageBytes(entrySizes, nextCid !== undefined));
+            try {
+                cids[pageNum] = await this._buildAndAddPage({ entries, nextCid, serialized, maximumPageSize, sortName, pageNum });
+            }
+            finally {
+                release();
+            }
         }
-        return { [sortName]: { pages: listOfPage, cids } };
+        if (!embedFirst)
+            return { [sortName]: { cids: cids } };
+        const firstPage = { comments: source.resolve(chunks[0]) };
+        if (cids[1])
+            firstPage.nextCid = cids[1]; // else every comment fit in the embedded page, nothing to continue to
+        return { [sortName]: { firstPage, cids: cids } };
     }
-    async addPreloadedCommentChunksToIpfs(chunks, sortName) {
-        const listOfPage = new Array(chunks.length);
-        const cids = [undefined]; // pageCids will never have the cid of preloaded page
-        const ipfsClient = this._community._clientsManager.getDefaultKuboRpcClient();
-        for (let pageNum = chunks.length - 1; pageNum >= 1; pageNum--) {
-            const pageIpfs = { nextCid: cids[pageNum + 1], comments: chunks[pageNum] };
-            if (!pageIpfs.nextCid)
-                delete pageIpfs.nextCid; // we don't to include undefined anywhere in the protocol
-            const maximumPageSize = 1024 * 1024 * Math.pow(2, Math.max(pageNum - 1, 0));
-            const stringifiedPageIpfs = deterministicStringify(pageIpfs);
-            const calculatedSizeOfStringifedPageIpfs = await calculateStringSizeSameAsIpfsAddCidV0(stringifiedPageIpfs);
-            if (calculatedSizeOfStringifedPageIpfs > maximumPageSize)
-                throw new PKCError("ERR_PAGE_GENERATED_IS_OVER_EXPECTED_SIZE", {
-                    calculatedSizeOfStringifedPageIpfs,
-                    pageIpfs,
-                    maximumPageSize,
-                    sortName,
-                    pageNum
-                });
-            const addRes = await retryKuboIpfsAddAndProvide({
-                ipfsClient: ipfsClient._client,
-                log: Logger("pkc-js:page-generator:addPreloadedCommentChunksToIpfs"),
-                content: stringifiedPageIpfs,
-                addOptions: { pin: true },
-                provideOptions: { recursive: true },
-                provideInBackground: true
+    // Build one page's string and add it. Its own function on purpose: a suspended async function keeps its
+    // registers, so a page string built inside the loop of _addChunksToIpfs would stay alive while the loop waits for
+    // the next page's budget; here it dies with this frame as soon as the add returns (issue #351).
+    async _buildAndAddPage({ entries, nextCid, serialized, maximumPageSize, sortName, pageNum }) {
+        const content = PAGE_OPEN + serialized.stringsOf(entries).join(",") + (nextCid ? pageCloseWithNextCid(nextCid) : PAGE_CLOSE);
+        const addRes = await retryKuboIpfsAddAndProvide({
+            ipfsClient: this._community._clientsManager.getDefaultKuboRpcClient()._client,
+            log: Logger("pkc-js:page-generator:addChunksToIpfs"),
+            content,
+            addOptions: { pin: true },
+            provideOptions: { recursive: true },
+            provideInBackground: true
+        });
+        if (addRes.size > maximumPageSize)
+            throw new PKCError("ERR_PAGE_GENERATED_IS_OVER_EXPECTED_SIZE", {
+                addRes,
+                expectedSize: maximumPageSize,
+                commentCount: entries.length,
+                sortName,
+                pageNum
             });
-            if (addRes.size > maximumPageSize)
-                throw new PKCError("ERR_PAGE_GENERATED_IS_OVER_EXPECTED_SIZE", {
-                    addRes,
-                    pageIpfs,
-                    maximumPageSize,
-                    sortName,
-                    pageNum
-                });
-            cids[pageNum] = addRes.path;
-            listOfPage[pageNum] = pageIpfs;
-        }
-        const firstPage = { comments: chunks[0], nextCid: cids[1] };
-        if (!firstPage.nextCid)
-            throw Error("First page should have nextCid");
-        listOfPage[0] = firstPage;
-        return { [sortName]: { pages: listOfPage, cids } };
+        return addRes.path;
     }
-    async _chunkComments({ comments, firstPageSizeBytes }) {
+    // Split comments into pages: the first under firstPageSizeBytes, then 1, 2, 4... MiB, each 1 KiB under its limit.
+    // A comment counts as what `ipfs add` would report for its JSON alone, so the sum overestimates the page by a few
+    // bytes per comment; `sizeOf` gives an entry's UTF-8 JSON size when the caller already serialized it (the page
+    // generator does, once per entry for every sort), else each comment is stringified here.
+    _chunkComments({ comments, firstPageSizeBytes, sizeOf }) {
         const FIRST_PAGE_SIZE = firstPageSizeBytes; // dynamic page size for preloaded sorts, 1MB for others
-        const SAFETY_MARGIN = 1024; // Use 1KiB margin
-        // Calculate overhead with and without nextCid
-        const OBJECT_WRAPPER_WITH_CID = (await calculateStringSizeSameAsIpfsAddCidV0(JSON.stringify({
-            comments: [],
-            nextCid: "QmXsYKgNH7XoZXdLko5uDvtWSRNE2AXuQ4u8KxVpCacrZx" // random cid as a place holder
-        }))) - 2; // Subtract 2 for empty array "[]"
-        const OBJECT_WRAPPER_WITHOUT_CID = (await calculateStringSizeSameAsIpfsAddCidV0(JSON.stringify({
-            comments: []
-        }))) - 2; // Subtract 2 for empty array "[]"
+        const bytesOf = sizeOf ?? ((entry) => Buffer.byteLength(JSON.stringify(entry)));
+        const entryBytes = new Array(comments.length);
+        for (let i = 0; i < comments.length; i++)
+            entryBytes[i] = bytesOf(comments[i]);
         // Quick check for small arrays - if everything fits in one page, no nextCid needed
-        const totalSizeWithoutCid = await calculateStringSizeSameAsIpfsAddCidV0(JSON.stringify({ comments }));
-        if (totalSizeWithoutCid <= FIRST_PAGE_SIZE) {
+        if (pageDagSize(entryBytes, false) <= FIRST_PAGE_SIZE)
             return [comments]; // Single page, no chunking needed
-        }
+        // The wrapper's cost with and without nextCid, minus the 2 bytes of the empty array
+        const OBJECT_WRAPPER_WITH_CID = pageDagSize([], true) - 2;
+        const OBJECT_WRAPPER_WITHOUT_CID = pageDagSize([], false) - 2;
         const chunks = [];
         let currentChunk = [];
         let chunkIndex = 0;
         let accumulatedSize = OBJECT_WRAPPER_WITH_CID;
-        // Pre-calculate sizes to avoid repeated stringification
-        const commentSizes = new Map();
-        async function getCommentSize(index) {
-            if (!commentSizes.has(index)) {
-                const size = await calculateStringSizeSameAsIpfsAddCidV0(JSON.stringify(comments[index]));
-                commentSizes.set(index, size);
-            }
-            return commentSizes.get(index);
-        }
-        function getCurrentMaxSize(index) {
-            if (index === 0) {
-                return FIRST_PAGE_SIZE; // First page is dynamic for preloaded
-            }
-            else {
-                const MB = 1024 * 1024;
-                // For preloaded: dynamic page size, 1MB, 2MB, 4MB, etc.
-                // For non-preloaded: 1MB, 2MB, 4MB, 8MB, etc.
-                return MB * Math.pow(2, index - 1); // index-1 because we want to start with 1MB
-            }
-        }
+        // The first page is dynamic; the pages after it 1MB, 2MB, 4MB...
+        const getCurrentMaxSize = (index) => (index === 0 ? FIRST_PAGE_SIZE : MB * Math.pow(2, index - 1));
         for (let i = 0; i < comments.length; i++) {
-            const commentSize = await getCommentSize(i);
+            const commentSize = calculateUnixFsDagSizeCidV0(entryBytes[i]);
             const maxSize = getCurrentMaxSize(chunkIndex);
             const isLastItem = i === comments.length - 1;
             // Add comma if needed
@@ -188,12 +319,7 @@ export class PageGenerator {
                     chunks.push(currentChunk);
                     currentChunk = [];
                     chunkIndex++;
-                    if (isLastItem) {
-                        accumulatedSize = OBJECT_WRAPPER_WITHOUT_CID;
-                    }
-                    else {
-                        accumulatedSize = OBJECT_WRAPPER_WITH_CID;
-                    }
+                    accumulatedSize = isLastItem ? OBJECT_WRAPPER_WITHOUT_CID : OBJECT_WRAPPER_WITH_CID;
                 }
                 else if (commentSize > maxSize - SAFETY_MARGIN) {
                     const log = Logger("pkc-js:page-generator:_chunkComments");
@@ -204,111 +330,272 @@ export class PageGenerator {
             currentChunk.push(comments[i]);
             accumulatedSize += commaSize + commentSize;
         }
-        if (currentChunk.length > 0) {
+        if (currentChunk.length > 0)
             chunks.push(currentChunk);
-        }
         return chunks;
     }
-    async sortAndChunkComments(unsortedComments, sortName, options) {
+    // Apply one sort to a loaded comment set: pinned placement, the maxAge window, then the file's `score` per
+    // comment, which drops what it declines.
+    // The exclusions were already applied by the SQL that loaded the set, and to `replies` (the scope's descendants,
+    // loaded once per generation and only for a file that declares requireReplies).
+    sortComments(comments, sort, baseTimestamp, replies, stripEntry, streamReplies) {
+        const { options } = sort;
+        const { pinned, unpinned } = partitionPageCommentsForSort({ comments, reserved: sort, baseTimestamp });
+        if (pinned.length + unpinned.length === 0)
+            return [];
+        // a requireReplies post sort scores pinned then unpinned in this order, and the provider streams subtrees along it
+        const descendantsOf = streamReplies && sort.file.requireReplies ? streamReplies([...pinned, ...unpinned]) : undefined;
+        return orderPageCommentsByScore({
+            pinned,
+            unpinned,
+            sortName: sort.sortName,
+            scoreOf: scorePageCommentsWithFile({ file: sort.file, options, baseTimestamp, replies, stripEntry, descendantsOf })
+        });
+    }
+    async sortAndChunkComments(unsortedComments, sort, options) {
         if (unsortedComments.length === 0)
             throw Error("Should not provide empty array of comments to sort");
-        const sortProps = options.parentCid
-            ? POST_REPLIES_SORT_TYPES[sortName]
-            : POSTS_SORT_TYPES[sortName];
-        if (typeof sortProps.score !== "function")
-            throw Error(`SortProps[${sortName}] score function is not defined`);
-        const scoreSort = (obj1, obj2) => {
-            // calculated from DB
-            if (sortName === "active") {
-                if (typeof obj1.activeScore !== "number")
-                    throw Error("Active score is not defined");
-                if (typeof obj2.activeScore !== "number")
-                    throw Error("Active score is not defined");
-                return obj2.activeScore - obj1.activeScore;
-            }
-            else {
-                const score1 = sortProps.score(obj1);
-                const score2 = sortProps.score(obj2);
-                return score2 - score1;
-            }
-        };
-        const pinnedComments = unsortedComments.filter((obj) => obj.commentUpdate.pinned === true).sort(scoreSort);
-        let unpinnedComments = unsortedComments.filter((obj) => !obj.commentUpdate.pinned).sort(scoreSort);
-        if (sortProps.timeframe) {
-            const timestampLower = options.baseTimestamp - TIMEFRAMES_TO_SECONDS[sortProps.timeframe];
-            unpinnedComments = unpinnedComments.filter((obj) => obj.comment.timestamp >= timestampLower);
-        }
-        const commentsSorted = pinnedComments.concat(unpinnedComments).map((comment) => omit(comment, ["activeScore"]));
+        const commentsSorted = this.sortComments(unsortedComments, sort, options.baseTimestamp, options.replies, options.stripEntry, options.streamReplies);
         if (commentsSorted.length === 0)
             return [];
-        const commentsChunks = await this._chunkComments({
-            comments: commentsSorted,
-            firstPageSizeBytes: options.firstPageSizeBytes
-        });
-        return commentsChunks;
+        return this._chunkComments({ comments: commentsSorted, firstPageSizeBytes: options.firstPageSizeBytes, sizeOf: options.sizeOf });
     }
-    // Resolves to sortedComments
-    // this is for non preloaded sorts
-    async sortChunkAddIpfsNonPreloaded(comments, sortName, options) {
-        const commentsChunks = await this.sortAndChunkComments(comments, sortName, options);
-        if (commentsChunks.length === 0)
-            return undefined;
-        const res = await this.addCommentChunksToIpfs(commentsChunks, sortName);
-        return res;
+    // The reply set requireReplies sorts score over: one unfiltered query per generation, shared by every such sort,
+    // each filtering it with its own exclusion options in JS. Nothing runs unless a sort asks.
+    _createRepliesLoader(load) {
+        let all;
+        const filtered = new Map();
+        return (sort) => {
+            all ??= load();
+            const key = JSON.stringify(sort.exclusions);
+            if (!filtered.has(key))
+                filtered.set(key, applyPageSortExclusions({ comments: all, exclusions: sort.exclusions, communityAddress: this._community.address }));
+            return filtered.get(key);
+        };
     }
-    _generationResToPages(res) {
-        const filteredGeneratedPages = res.filter(Boolean); // Take out undefined values
-        if (filteredGeneratedPages.length === 0)
-            return undefined;
-        const mergedObject = Object.assign({}, ...filteredGeneratedPages);
-        return {
-            pages: Object.assign({}, ...Object.entries(mergedObject).map(([sortName, pages]) => ({ [sortName]: pages.pages[0] }))),
-            pageCids: Object.assign({}, ...Object.entries(mergedObject).map(([sortName, pages]) => (pages.cids?.[0] ? { [sortName]: pages.cids[0] } : undefined))),
-            allPageCids: Object.assign({}, ...Object.entries(mergedObject).map(([sortName, data]) => data.cids ? { [sortName]: data.cids.filter((c) => typeof c === "string") } : undefined))
+    // The reply set a requireReplies post sort scores over, streamed: the provider walks the posts in the order the
+    // scorer asks for them and loads one batch of posts' subtrees at a time (lean entries, the sort's exclusions
+    // applied), so a board's replies are never in memory at once; the file still receives each post's whole subtree
+    // (issue #351). A post asked out of order (never, in practice) just loads its own batch.
+    _createStreamingRepliesProvider(sort) {
+        const db = this._community._dbHandler;
+        const communityAddress = this._community.address;
+        return (ordered) => {
+            const indexOf = new Map(ordered.map((entry, index) => [entry.commentUpdate.cid, index]));
+            let batchStart = -1;
+            let batchEnd = -1;
+            let descendantsOf;
+            return (cid) => {
+                const index = indexOf.get(cid);
+                if (index === undefined)
+                    return [];
+                if (index < batchStart || index >= batchEnd || !descendantsOf) {
+                    batchStart = index;
+                    batchEnd = Math.min(index + SERIALIZE_BATCH, ordered.length);
+                    const replies = db.queryAllRepliesForPageSort({
+                        postCids: ordered.slice(batchStart, batchEnd).map((entry) => entry.commentUpdate.cid)
+                    });
+                    descendantsOf = createDescendantsLookup(applyPageSortExclusions({ comments: replies, exclusions: sort.exclusions, communityAddress }));
+                }
+                return descendantsOf(cid);
+            };
         };
     }
-    async generateCommunityPosts(preloadedPageSortName, preloadedPageSizeBytes) {
-        const pageOptions = {
-            excludeCommentsWithDifferentCommunityAddress: true,
-            excludeDeletedComments: true,
-            excludeRemovedComments: true,
-            excludeCommentPendingApproval: true,
-            excludeCommentWithApprovedFalse: true,
-            parentCid: null,
-            preloadedPage: preloadedPageSortName,
-            baseTimestamp: timestamp(),
-            firstPageSizeBytes: preloadedPageSizeBytes
+    // Load the comment set each sort runs over. Sorts that share the same exclusion options (all of them, unless an
+    // owner set an exclude* option on one entry) share one query; flat sorts have their own query shape.
+    _createCommentLoader(load) {
+        const cache = new Map();
+        const loader = (sort) => {
+            const exclusions = sort.exclusions;
+            const key = `${sort.flat}:${JSON.stringify(exclusions)}`;
+            if (!cache.has(key))
+                cache.set(key, load(exclusions, sort.flat));
+            return cache.get(key);
         };
-        // Sorting posts on a community level
-        const rawPostsUnresolved = this._community._dbHandler.queryPostsWithActiveScore(pageOptions);
-        if (rawPostsUnresolved.length === 0)
+        loader.loaded = () => [...cache.values()].flat(); // every entry any sort of this generation loaded
+        return loader;
+    }
+    // Generic generation for one scope (community posts, or one comment's replies) from its configured sorts.
+    //
+    // 1. Every sort is applied to its comment set; a sort whose file throws is recorded in failedSorts and skipped so
+    //    the remaining sorts still publish. When that leaves no pages at all the generation throws
+    //    ERR_ALL_PAGE_SORTS_FAILED_TO_GENERATE (failedSorts in its details) instead of returning "nothing to
+    //    paginate": publishing an empty scope would replace a live feed on a sort file's bug. Loading the comment
+    //    set is pkc-js's own code, so an error there (a busy sqlite, a malformed row) propagates as well. Either
+    //    way the cycle aborts, the last good record stays published and the sync loop retries next cycle.
+    // 2. Preloaded sorts split preloadedPageSizeBytes equally. One whose first chunk does not fit its share drops to
+    //    pageCids (it is generated like a non-preloaded sort) while the others still embed.
+    // 3. If every preloaded sort fits in a single chunk, the single-page shortcut applies: only those pages are returned
+    //    and nothing else is generated (docs/protocol/pages.md, "single-chunk shortcut"). Skipping the other sorts is
+    //    only sound when a client holding the embedded pages can build them locally, so when any sort would be
+    //    skipped (a non-preloaded or demoted one) every embedded page must hold the whole comment set: a windowed or
+    //    filtered preloaded sort next to other sorts takes the full path instead. With no preloaded sort at all there
+    //    is no shortcut and every sort goes to pageCids.
+    // 4. Otherwise preloaded sorts embed their first chunk and add the rest to IPFS; the others add every chunk.
+    // Results keep the configured order so `pages` keys tell a client which preloaded sort is the default.
+    //
+    // The comment sets are lean: an entry carries its DB-format reply refs, which `score` never sees since `replies`
+    // is stripped. What a page holds for an entry comes from `source`, which resolves a batch's nested reply pages
+    // from rows; entries are sized through SerializedEntries (serialized a batch at a time, strings kept in the
+    // entry cache while its budget allows) and fetched per page when built, so a sort costs an ordered list of
+    // references, and a board's reply trees stream through instead of being held at once (issue #351).
+    async _generatePagesForSorts({ scope, sorts, loadComments, loadReplies, streamReplies, source, entryCache, preloadedPageSizeBytes, baseTimestamp, parentCid }) {
+        if (sorts.length === 0)
             return undefined;
-        // Resolve CID-ref replies for each post so pages have full nested reply trees
-        const rawPosts = this._community._dbHandler.resolveRepliesCidRefsForEntries(rawPostsUnresolved);
-        const preloadedChunk = await this.sortAndChunkComments(rawPosts, preloadedPageSortName, pageOptions);
-        const firstChunkSize = await getSerializedCommentsSize(preloadedChunk[0], preloadedChunk.length > 1);
-        if (firstChunkSize > preloadedPageSizeBytes)
-            throw new PKCError("ERR_PAGE_GENERATED_IS_OVER_EXPECTED_SIZE", {
-                firstChunkSize,
-                preloadedPageSizeBytes,
-                sortName: preloadedPageSortName
+        const failedSorts = {};
+        const fail = (sort, error) => {
+            failedSorts[sort.sortName] = new PKCError("ERR_PAGE_SORT_FAILED_TO_GENERATE", {
+                sortName: sort.sortName,
+                scope,
+                pageSortName: sort.settings.name ?? sort.settings.path,
+                error
             });
-        if (preloadedChunk.length === 1)
-            return { singlePreloadedPage: { [preloadedPageSortName]: { comments: preloadedChunk[0] } } }; // all comments fit in one preloaded page
-        // we're gonna have pages for each sort type, they don't fit in a single preloaded chunk
-        const sortResults = [];
-        sortResults.push(await this.addPreloadedCommentChunksToIpfs(preloadedChunk, preloadedPageSortName));
-        const nonPreloadedSorts = keys(POSTS_SORT_TYPES).filter((sortName) => sortName !== preloadedPageSortName);
-        await Promise.all(nonPreloadedSorts.map(async (sortName) => {
-            sortResults.push(await this.sortChunkAddIpfsNonPreloaded(rawPosts, sortName, {
-                ...pageOptions,
-                firstPageSizeBytes: 1024 * 1024
-            }));
-        }));
-        const generatedPages = this._generationResToPages(sortResults);
+        };
+        // Load first so the preload budget is split only among preloaded sorts that have something to embed. Not
+        // inside the per-sort net: only the sort file's own code may fail a single sort (see 1. above).
+        const loaded = [];
+        for (const sort of sorts) {
+            const comments = loadComments(sort);
+            if (comments.length > 0)
+                loaded.push({ sort, comments, ...(sort.file.requireReplies && loadReplies ? { replies: loadReplies(sort) } : {}) });
+        }
+        if (loaded.length === 0)
+            return undefined;
+        const serialized = new SerializedEntries(source, entryCache);
+        for (const { comments } of loaded)
+            serialized.sizeAll(comments); // one serialization per entry, shared by every sort
+        const strippedEntries = new Map(); // what `score` receives, built once per entry
+        const stripEntry = (entry) => {
+            let stripped = strippedEntries.get(entry);
+            if (!stripped)
+                strippedEntries.set(entry, (stripped = stripRepliesFromPageComment(entry)));
+            return stripped;
+        };
+        const preloadedCount = loaded.filter(({ sort }) => sort.preloaded).length;
+        const share = preloadedCount > 0 ? Math.floor(preloadedPageSizeBytes / preloadedCount) : 0;
+        const preloaded = [];
+        const nonPreloaded = [];
+        for (const { sort, comments, replies } of loaded) {
+            let chunks;
+            try {
+                chunks = await this.sortAndChunkComments(comments, sort, {
+                    baseTimestamp,
+                    parentCid,
+                    firstPageSizeBytes: sort.preloaded ? share : NON_PRELOADED_FIRST_PAGE_SIZE,
+                    replies,
+                    sizeOf: serialized.sizeOf,
+                    stripEntry,
+                    streamReplies: streamReplies?.(sort)
+                });
+            }
+            catch (e) {
+                fail(sort, e);
+                continue;
+            }
+            if (chunks.length === 0)
+                continue; // every comment filtered out
+            if (!sort.preloaded) {
+                nonPreloaded.push({ sort, chunks });
+                continue;
+            }
+            const firstChunkSize = pageDagSize(chunks[0].map(serialized.sizeOf), chunks.length > 1);
+            if (firstChunkSize > share) {
+                // Its share of the budget is not enough even for its first chunk: degrade this sort to pageCids
+                const rechunked = this._chunkComments({
+                    comments: chunks.flat(),
+                    firstPageSizeBytes: NON_PRELOADED_FIRST_PAGE_SIZE,
+                    sizeOf: serialized.sizeOf
+                });
+                nonPreloaded.push({ sort, chunks: rechunked });
+            }
+            else
+                preloaded.push({
+                    sort,
+                    chunks,
+                    holdsWholeSet: chunks.length === 1 && chunks[0].length === comments.length
+                });
+        }
+        if (preloaded.length === 0 && nonPreloaded.length === 0) {
+            // Nothing to publish for this scope. With a sort file to blame that is not a verdict on the feed (the sort may
+            // throw on every comment), so the cycle aborts and the last published pages stay up (see 1. above).
+            if (Object.keys(failedSorts).length > 0)
+                throw new PKCError("ERR_ALL_PAGE_SORTS_FAILED_TO_GENERATE", { scope, parentCid, failedSorts });
+            return undefined;
+        }
+        const everyPreloadedIsSingleChunk = preloaded.length > 0 && preloaded.every(({ chunks }) => chunks.length === 1);
+        const skippingOtherSortsIsSound = nonPreloaded.length === 0 || preloaded.every(({ holdsWholeSet }) => holdsWholeSet);
+        if (everyPreloadedIsSingleChunk && skippingOtherSortsIsSound) {
+            const singlePreloadedPage = {};
+            for (const { sort, chunks } of preloaded)
+                singlePreloadedPage[sort.sortName] = { comments: source.resolve(chunks[0]) };
+            return { singlePreloadedPage, failedSorts };
+        }
+        // Full generation. Preloaded first, then the others, each in configured order; IPFS adds run in parallel
+        // under one byte budget for the pages in flight.
+        const budget = new PageBytesBudget(serializedEntryCacheBudgetBytes());
+        const results = await Promise.all([
+            ...preloaded.map(({ sort, chunks }) => this._addChunksToIpfs({ chunks, sortName: sort.sortName, serialized, embedFirst: true, source, budget })),
+            ...nonPreloaded.map(({ sort, chunks }) => this._addChunksToIpfs({ chunks, sortName: sort.sortName, serialized, embedFirst: false, source, budget }))
+        ]);
+        const generatedPages = this._generationResToPages(results);
         if (!generatedPages)
             return undefined;
-        return generatedPages;
+        return { ...generatedPages, failedSorts };
+    }
+    _generationResToPages(res) {
+        const filteredGeneratedPages = res.filter(Boolean); // Take out undefined values
+        if (filteredGeneratedPages.length === 0)
+            return undefined;
+        const mergedObject = Object.assign({}, ...filteredGeneratedPages);
+        const pages = {};
+        const pageCids = {};
+        const allPageCids = {};
+        for (const [sortName, data] of Object.entries(mergedObject)) {
+            if (!data)
+                continue;
+            const firstCid = data.cids[0];
+            if (firstCid)
+                pageCids[sortName] = firstCid; // pageCids never carries a preloaded sort's first page
+            else {
+                assert(data.firstPage, `preloaded sort ${sortName} has no embedded first page`);
+                pages[sortName] = data.firstPage;
+            }
+            const cids = data.cids;
+            allPageCids[sortName] = cids.filter((c) => typeof c === "string");
+        }
+        return { pages, ...(Object.keys(pageCids).length > 0 ? { pageCids } : {}), allPageCids };
+    }
+    async _pageSortsFor(scope) {
+        return (await this._community._ensurePageSortsLoaded())[scope];
+    }
+    // Page entries out of the community database: each batch's nested reply pages resolved from rows. Plain
+    // JSON.stringify is canonical here: the row mapper builds every object with sorted keys
+    // (createPositionalCommentRowMapper), so this is byte-identical to safe-stable-stringify at native speed.
+    _dbPageEntrySource() {
+        const db = this._community._dbHandler;
+        const resolve = (entries) => db.resolveRepliesCidRefsForEntries(entries);
+        return { resolve, serialize: (entries) => resolve(entries).map((entry) => JSON.stringify(entry)) };
+    }
+    async generateCommunityPosts({ preloadedPageSizeBytes }) {
+        const baseTimestamp = timestamp();
+        const sorts = await this._pageSortsFor("posts");
+        // Posts are loaded lean, their reply pages still as CID refs; the page entry source resolves them per batch
+        // (issue #351)
+        const loadComments = this._createCommentLoader((exclusions) => this._community._dbHandler.queryPosts({ ...exclusions, parentCid: null }));
+        this._postEntryCache ??= new EntryJsonCache(serializedEntryCacheBudgetBytes());
+        const generated = await this._generatePagesForSorts({
+            scope: "posts",
+            sorts,
+            loadComments,
+            streamReplies: (sort) => this._createStreamingRepliesProvider(sort), // a requireReplies post sort streams each post's subtree
+            source: this._dbPageEntrySource(),
+            entryCache: this._postEntryCache,
+            preloadedPageSizeBytes,
+            baseTimestamp,
+            parentCid: null
+        });
+        this._postEntryCache.retainOnly(new Set(loadComments.loaded().map((entry) => entry.commentUpdate.cid)));
+        return generated;
     }
     async _bundleLatestCommentUpdateWithQueuedComments(queuedComment) {
         const communityAuthor = this._community._dbHandler.queryCommunityAuthor(queuedComment.authorSignerAddress);
@@ -340,107 +627,43 @@ export class PageGenerator {
             return undefined;
         const queuedComments = await Promise.all(commentsPendingApproval.map((comment) => this._bundleLatestCommentUpdateWithQueuedComments(comment)));
         const combinedHashOfCids = sha256(queuedComments.map((comment) => comment.commentUpdate.cid).join(""));
-        const chunkedQueuedComments = await this._chunkComments({ comments: queuedComments, firstPageSizeBytes });
+        const chunkedQueuedComments = this._chunkComments({ comments: queuedComments, firstPageSizeBytes });
         const pages = await this.addQueuedCommentChunksToIpfs(chunkedQueuedComments, "pendingApproval");
         return { pageCids: { pendingApproval: pages.cids[0] }, combinedHashOfCids };
     }
-    async generatePostPages(comment, preloadedReplyPageSortName, preloadedPageSizeBytes) {
-        const pageOptions = {
-            excludeCommentsWithDifferentCommunityAddress: true,
-            excludeDeletedComments: false,
-            excludeRemovedComments: false,
-            excludeCommentWithApprovedFalse: false,
-            excludeCommentPendingApproval: true,
-            parentCid: comment.cid,
-            preloadedPage: preloadedReplyPageSortName,
-            baseTimestamp: timestamp()
-        };
-        // Use recursive query that follows CID-ref lists in DB replies to build nested trees
-        const hierarchalReplies = this._community._dbHandler.queryPageCommentsWithResolvedReplies(pageOptions);
-        if (hierarchalReplies.length === 0)
-            return undefined;
-        const preloadedChunk = await this.sortAndChunkComments(hierarchalReplies, preloadedReplyPageSortName, {
-            ...pageOptions,
-            firstPageSizeBytes: preloadedPageSizeBytes
+    // Reply pages of one comment. Flat sorts (flattened descendant subtree) are generated for post replies only:
+    // depth-1+ comments have never had them and one settings.pages.replies list applies at every depth.
+    async _generateRepliesPages(comment, preloadedPageSizeBytes, includeFlatSorts) {
+        const baseTimestamp = timestamp();
+        const sorts = (await this._pageSortsFor("replies")).filter((sort) => includeFlatSorts || !sort.flat);
+        const loadComments = this._createCommentLoader((exclusions, flat) => {
+            const pageOptions = { ...exclusions, parentCid: comment.cid, baseTimestamp };
+            return flat
+                ? this._community._dbHandler.queryFlattenedPageReplies({ ...pageOptions, commentUpdateFieldsToExclude: ["replies"] })
+                : this._community._dbHandler.queryPageCommentsWithResolvedReplies(pageOptions); // recursive query following CID-ref lists in DB replies to build nested trees
         });
-        const firstChunkSize = await getSerializedCommentsSize(preloadedChunk[0], preloadedChunk.length > 1);
-        const disablePreload = firstChunkSize > preloadedPageSizeBytes;
-        if (!disablePreload && preloadedChunk.length === 1)
-            return { singlePreloadedPage: { [preloadedReplyPageSortName]: { comments: preloadedChunk[0] } } }; // all comments fit in one page
-        const sortResults = [];
-        if (disablePreload) {
-            sortResults.push(await this.sortChunkAddIpfsNonPreloaded(hierarchalReplies, preloadedReplyPageSortName, {
-                ...pageOptions,
-                firstPageSizeBytes: 1024 * 1024
-            }));
-        }
-        else {
-            sortResults.push(await this.addPreloadedCommentChunksToIpfs(preloadedChunk, preloadedReplyPageSortName));
-        }
-        const nonPreloadedSorts = keys(POST_REPLIES_SORT_TYPES).filter((sortName) => sortName !== preloadedReplyPageSortName);
-        const flattenedReplies = this._community._dbHandler.queryFlattenedPageReplies({
-            ...pageOptions,
-            commentUpdateFieldsToExclude: ["replies"]
+        // Every reply under the comment's post, for a requireReplies reply sort; each reply's own subtree is sliced from it
+        const loadReplies = this._createRepliesLoader(() => {
+            const postCid = comment.postCid ?? this._community._dbHandler.queryComment(comment.cid)?.postCid ?? comment.cid;
+            return this._community._dbHandler.queryAllRepliesForPageSort({ postCid });
         });
-        await Promise.all(nonPreloadedSorts.map(async (sortName) => {
-            const replies = POST_REPLIES_SORT_TYPES[sortName].flat ? flattenedReplies : hierarchalReplies;
-            sortResults.push(await this.sortChunkAddIpfsNonPreloaded(replies, sortName, { ...pageOptions, firstPageSizeBytes: 1024 * 1024 }));
-        }));
-        const generatedPages = this._generationResToPages(sortResults);
-        if (!generatedPages)
-            return undefined;
-        if (disablePreload)
-            return { pageCids: generatedPages.pageCids, pages: {}, allPageCids: generatedPages.allPageCids };
-        else
-            return generatedPages;
-    }
-    async generateReplyPages(comment, preloadedReplyPageSortName, preloadedPageSizeBytes) {
-        const pageOptions = {
-            excludeCommentsWithDifferentCommunityAddress: true,
-            excludeDeletedComments: false,
-            excludeRemovedComments: false,
-            excludeCommentPendingApproval: true,
-            parentCid: comment.cid,
-            preloadedPage: preloadedReplyPageSortName,
-            baseTimestamp: timestamp(),
-            excludeCommentWithApprovedFalse: false
-        };
-        // Use recursive query that follows CID-ref lists in DB replies to build nested trees
-        const hierarchalReplies = this._community._dbHandler.queryPageCommentsWithResolvedReplies(pageOptions);
-        if (hierarchalReplies.length === 0)
-            return undefined;
-        const preloadedChunk = await this.sortAndChunkComments(hierarchalReplies, preloadedReplyPageSortName, {
-            ...pageOptions,
-            firstPageSizeBytes: preloadedPageSizeBytes
+        return this._generatePagesForSorts({
+            scope: "replies",
+            sorts,
+            loadComments,
+            loadReplies,
+            source: this._dbPageEntrySource(),
+            entryCache: new EntryJsonCache(serializedEntryCacheBudgetBytes()), // one comment's replies, dropped with the generation
+            preloadedPageSizeBytes,
+            baseTimestamp,
+            parentCid: comment.cid
         });
-        const firstChunkSize = await getSerializedCommentsSize(preloadedChunk[0], preloadedChunk.length > 1);
-        const disablePreload = firstChunkSize > preloadedPageSizeBytes;
-        if (!disablePreload && preloadedChunk.length === 1)
-            return { singlePreloadedPage: { [preloadedReplyPageSortName]: { comments: preloadedChunk[0] } } }; // all comments fit in one page
-        const nonPreloadedSorts = keys(REPLY_REPLIES_SORT_TYPES).filter((sortName) => sortName !== preloadedReplyPageSortName);
-        const sortResults = [];
-        if (disablePreload) {
-            sortResults.push(await this.sortChunkAddIpfsNonPreloaded(hierarchalReplies, preloadedReplyPageSortName, {
-                ...pageOptions,
-                firstPageSizeBytes: 1024 * 1024
-            }));
-        }
-        else {
-            sortResults.push(await this.addPreloadedCommentChunksToIpfs(preloadedChunk, preloadedReplyPageSortName));
-        }
-        await Promise.all(nonPreloadedSorts.map(async (hierarchalSortName) => {
-            sortResults.push(await this.sortChunkAddIpfsNonPreloaded(hierarchalReplies, hierarchalSortName, {
-                ...pageOptions,
-                firstPageSizeBytes: 1024 * 1024 // pageCids will always have first pages with limit of 1mib, regardless of preloadedPageSizeBytes
-            }));
-        }));
-        const generatedPages = this._generationResToPages(sortResults);
-        if (!generatedPages)
-            return undefined;
-        if (disablePreload)
-            return { pageCids: generatedPages.pageCids, pages: {}, allPageCids: generatedPages.allPageCids };
-        else
-            return generatedPages;
+    }
+    async generatePostPages(comment, preloadedPageSizeBytes) {
+        return this._generateRepliesPages(comment, preloadedPageSizeBytes, true);
+    }
+    async generateReplyPages(comment, preloadedPageSizeBytes) {
+        return this._generateRepliesPages(comment, preloadedPageSizeBytes, false);
     }
     toJSON() {
         return undefined;