@pkcprotocol/pkc-js 0.0.18 → 0.0.20

package/dist/node/runtime/node/community/local-community.js

@@ -2,6 +2,7 @@ import Logger from "../../../logger.js";
 import { LRUCache } from "lru-cache";
 import { PageGenerator } from "./page-generator.js";
 import { DbHandler } from "./db-handler.js";
+import { deriveDbReplies, deriveDbPosts, resolveDbPostsCidRefs } from "../util.js";
 import { of as calculateIpfsHash } from "typestub-ipfs-only-hash";
 import { derivePublicationFromChallengeRequest, doesDomainAddressHaveCapitalLetter, genToArray, hideClassPrivateProps, ipnsNameToIpnsOverPubsubTopic, isLinkOfMedia, isLinkOfImage, isLinkOfVideo, isLinkOfAnimatedImage, isLinkValid, isStringDomain, pubsubTopicToDhtKey, timestamp, getErrorCodeFromMessage, removeMfsFilesSafely, removeBlocksFromKuboNode, writeKuboFilesWithTimeout, retryKuboIpfsAddAndProvide, retryKuboBlockPutPinAndProvidePubsubTopic, calculateIpfsCidV0, calculateStringSizeSameAsIpfsAddCidV0, getIpnsRecordInLocalKuboNode, contentContainsMarkdownImages, contentContainsMarkdownVideos, isLinkOfAudio, contentContainsMarkdownAudio, areEquivalentCommunityAddresses } from "../../../util.js";
 import { STORAGE_KEYS } from "../../../constants.js";
@@ -18,7 +19,7 @@ import env from "../../../version.js";
 import { getIpfsKeyFromPrivateKey, getPKCAddressFromPublicKey, getPublicKeyFromPrivateKey } from "../../../signer/util.js";
 import { RpcLocalCommunity } from "../../../community/rpc-local-community.js";
 import * as remeda from "remeda";
-import { buildRuntimeAuthor, cleanWireAuthor, getAuthorDomainFromWire, getAuthorNameFromWire } from "../../../publications/publication-author.js";
+import { buildRuntimeAuthor, cleanWireAuthor, getAuthorNameFromWire } from "../../../publications/publication-author.js";
 import { getCommunityPublicKeyFromWire, getCommunityNameFromWire } from "../../../publications/publication-community.js";
 import { CommentEditPubsubMessagePublicationSchema, CommentEditPubsubMessagePublicationWithFlexibleAuthorSchema, CommentEditReservedFields } from "../../../publications/comment-edit/schema.js";
 import { CommunityIpfsSchema, CommunitySignedPropertyNames } from "../../../community/schema.js";
@@ -83,6 +84,7 @@ export class LocalCommunity extends RpcLocalCommunity {
         this._mirroredStartedOrUpdatingCommunity = undefined; // The pkc._startedCommunities we're subscribed to
         this._pendingEditProps = [];
         this._blocksToRm = [];
+        this._postsAllPageCids = undefined;
         this.handleChallengeExchange = this.handleChallengeExchange.bind(this);
         this._setState("stopped");
         this.started = false;
@@ -104,7 +106,10 @@ export class LocalCommunity extends RpcLocalCommunity {
         this._dbHandler = undefined;
         hideClassPrivateProps(this);
     }
-    // This will be stored in DB
+    // This will be stored in DB and also shared between instances via _updateInstancePropsWithStartedCommunityOrDb.
+    // Must NOT convert posts to CID-ref format here — that's only for DB storage (done in _updateDbInternalState).
+    // CID-ref conversion strips preloaded page data which breaks reply CommentUpdate resolution
+    // when other instances read the shared state.
     toJSONInternalAfterFirstUpdate() {
         const rpcJson = this.toJSONInternalRpcAfterFirstUpdate();
         return {
@@ -164,6 +169,29 @@ export class LocalCommunity extends RpcLocalCommunity {
             this.settings = newProps.settings;
     }
     async initInternalCommunityAfterFirstUpdateNoMerge(newProps) {
+        // Detect CID-ref format posts from DB: wire format always has 'pages' key, CID-ref format doesn't
+        if (newProps.posts && !("pages" in newProps.posts)) {
+            const dbPosts = newProps.posts;
+            // Extract allPageCids for future unpinning
+            const allPageCids = {};
+            for (const [sortName, entry] of Object.entries(dbPosts)) {
+                if (entry?.allPageCids?.length)
+                    allPageCids[sortName] = entry.allPageCids;
+            }
+            this._postsAllPageCids = Object.keys(allPageCids).length > 0 ? allPageCids : undefined;
+            // Lightweight conversion: just pageCids from allPageCids[0], preloaded pages regenerated on next update.
+            // Never use resolveDbPostsCidRefs here — this method is called from many code paths
+            // where _dbHandler._db may not be initialized (e.g. updateListener from a mirrored community).
+            const pageCids = {};
+            for (const [sortName, entry] of Object.entries(dbPosts)) {
+                if (entry?.allPageCids?.[0])
+                    pageCids[sortName] = entry.allPageCids[0];
+            }
+            newProps = {
+                ...newProps,
+                posts: { pages: {}, ...(Object.keys(pageCids).length > 0 ? { pageCids } : {}) }
+            };
+        }
         const keysOfCommunityIpfs = [...CommunitySignedPropertyNames, "signature"];
         this.initRpcInternalCommunityAfterFirstUpdateNoMerge({
             community: remeda.pick(newProps, keysOfCommunityIpfs),
@@ -288,7 +316,17 @@ export class LocalCommunity extends RpcLocalCommunity {
             await this._dbHandler.lockCommunityState();
             lockedIt = true;
             const internalStateBefore = await this._getDbInternalState(false);
-            const mergedInternalState = { ...internalStateBefore, ...props };
+            // Convert posts to CID-ref format for compact DB storage (strip preloaded page data)
+            const propsToStore = "posts" in props && props.posts
+                ? {
+                    ...props,
+                    posts: deriveDbPosts({
+                        posts: props.posts,
+                        allPageCids: this._postsAllPageCids
+                    })
+                }
+                : props;
+            const mergedInternalState = { ...internalStateBefore, ...propsToStore };
             await this._dbHandler.keyvSet(STORAGE_KEYS[STORAGE_KEYS.INTERNAL_COMMUNITY], mergedInternalState);
             this._internalStateUpdateId = props._internalStateUpdateId;
             log.trace("Updated community", this.address, "internal state in db with new props", Object.keys(props));
@@ -336,6 +374,13 @@ export class LocalCommunity extends RpcLocalCommunity {
     async _updateInstanceStateWithDbState() {
         const currentDbState = await this._getDbInternalState(false);
         if ("updatedAt" in currentDbState) {
+            // Resolve CID-ref posts from DB back to full wire format with preloaded pages.
+            // DB stores posts in compact CID-ref format (no preloaded page data).
+            // _dbHandler is guaranteed to be initialized here since we're loading from DB.
+            if (currentDbState.posts && !("pages" in currentDbState.posts)) {
+                const dbPosts = currentDbState.posts;
+                currentDbState.posts = resolveDbPostsCidRefs({ dbPosts, dbHandler: this._dbHandler });
+            }
             await this.initInternalCommunityAfterFirstUpdateNoMerge(currentDbState);
         }
         else
@@ -534,6 +579,8 @@ export class LocalCommunity extends RpcLocalCommunity {
         const availablePostsSize = MAX_FILE_SIZE_BYTES_FOR_COMMUNITY_IPFS - communityWithoutPostsSignatureSize - expectedSignatureSize - 1000;
         const generatedPosts = await this._pageGenerator.generateCommunityPosts(preloadedPostsPages, availablePostsSize);
         // posts should not be cleaned up because we want to make sure not to modify authors' posts
+        // Extract allPageCids from generation result for DB CID-ref storage and unpinning
+        const newPostsAllPageCids = generatedPosts && !("singlePreloadedPage" in generatedPosts) ? generatedPosts.allPageCids : undefined;
         if (generatedPosts) {
             if ("singlePreloadedPage" in generatedPosts)
                 newIpns.posts = { pages: generatedPosts.singlePreloadedPage };
@@ -547,9 +594,17 @@ export class LocalCommunity extends RpcLocalCommunity {
         }
         else {
             await this._updateDbInternalState({ posts: undefined }); // make sure db resets posts as well
-            // TODO make sure to capture this.posts cids to unpin
         }
-        this._addOldPageCidsToCidsToUnpin(this.raw.communityIpfs?.posts, newIpns.posts).catch((err) => log.error("Failed to add old page cids of community.posts to _cidsToUnpin", err));
+        // Unpin old posts page CIDs using direct allPageCids comparison (no IPFS fetches needed)
+        {
+            const oldCids = new Set(this._postsAllPageCids ? Object.values(this._postsAllPageCids).flat() : []);
+            const newCids = new Set(newPostsAllPageCids ? Object.values(newPostsAllPageCids).flat() : []);
+            for (const cid of oldCids) {
+                if (!newCids.has(cid))
+                    this._cidsToUnPin.add(cid);
+            }
+        }
+        this._postsAllPageCids = newPostsAllPageCids;
         if (newModQueue) {
             newIpns.modQueue = { pageCids: newModQueue.pageCids };
         }
@@ -779,12 +834,12 @@ export class LocalCommunity extends RpcLocalCommunity {
             // Check if the comment was published with pseudonymity - if so, get the original author address/domain
             const aliasInfo = this._dbHandler.queryPseudonymityAliasByCommentCid(commentModRaw.commentCid);
             if (aliasInfo) {
-                targetAuthorSignerAddress = await getPKCAddressFromPublicKey(aliasInfo.originalAuthorSignerPublicKey);
-                targetAuthorDomain = aliasInfo.originalAuthorDomain || undefined;
+                targetAuthorSignerAddress = await getPKCAddressFromPublicKey(aliasInfo.originalAuthorPublicKey);
+                targetAuthorDomain = aliasInfo.originalAuthorName || undefined;
             }
             else {
                 targetAuthorSignerAddress = commentToBeEdited.authorSignerAddress;
-                targetAuthorDomain = getAuthorDomainFromWire(commentToBeEdited.author);
+                targetAuthorDomain = getAuthorNameFromWire(commentToBeEdited.author);
             }
         }
         const modTableRow = {
@@ -905,7 +960,7 @@ export class LocalCommunity extends RpcLocalCommunity {
         if (opts.mode === "per-post") {
             // For a new post (no postCid yet), always generate a fresh alias; once stored the postCid will be used for reuse.
             if (opts.postCid) {
-                const existing = this._dbHandler.queryPseudonymityAliasForPost(opts.originalAuthorSignerPublicKey, opts.postCid);
+                const existing = this._dbHandler.queryPseudonymityAliasForPost(opts.originalAuthorPublicKey, opts.postCid);
                 if (existing?.aliasPrivateKey)
                     return existing.aliasPrivateKey;
             }
@@ -916,7 +971,7 @@ export class LocalCommunity extends RpcLocalCommunity {
             return signer.privateKey;
         }
         else if (opts.mode === "per-author") {
-            const existing = this._dbHandler.queryPseudonymityAliasForAuthor(opts.originalAuthorSignerPublicKey);
+            const existing = this._dbHandler.queryPseudonymityAliasForAuthor(opts.originalAuthorPublicKey);
             if (existing?.aliasPrivateKey)
                 return existing.aliasPrivateKey;
             const signer = await this._pkc.createSigner();
@@ -933,11 +988,11 @@ export class LocalCommunity extends RpcLocalCommunity {
         const isAuthorMod = await this._isPublicationAuthorPartOfRoles(originalComment, ["owner", "admin", "moderator"]);
         if (isAuthorMod)
             return { publication: originalComment };
-        const originalAuthorSignerPublicKey = originalComment.signature.publicKey;
+        const originalAuthorPublicKey = originalComment.signature.publicKey;
         const postCid = originalComment.postCid;
         const aliasPrivateKey = await this._resolveAliasPrivateKeyForCommentPublication({
             mode,
-            originalAuthorSignerPublicKey,
+            originalAuthorPublicKey,
             postCid
         });
         const aliasSigner = await this._pkc.createSigner({ privateKey: aliasPrivateKey, type: "ed25519" });
@@ -958,7 +1013,7 @@ export class LocalCommunity extends RpcLocalCommunity {
             publication: anonymizedComment,
             anonymity: {
                 aliasPrivateKey,
-                originalAuthorSignerPublicKey,
+                originalAuthorPublicKey,
                 mode,
                 originalComment
             }
@@ -1080,8 +1135,8 @@ export class LocalCommunity extends RpcLocalCommunity {
                     {
                         commentCid: storedComment.cid,
                         aliasPrivateKey: anonymity.aliasPrivateKey,
-                        originalAuthorSignerPublicKey: anonymity.originalAuthorSignerPublicKey,
-                        originalAuthorDomain: getAuthorDomainFromWire(anonymity.originalComment.author) || null,
+                        originalAuthorPublicKey: anonymity.originalAuthorPublicKey,
+                        originalAuthorName: getAuthorNameFromWire(anonymity.originalComment.author) || null,
                         mode: anonymity.mode,
                         insertedAt: timestamp()
                     }
@@ -1210,7 +1265,7 @@ export class LocalCommunity extends RpcLocalCommunity {
             }
             log("Returning idempotent success for duplicate comment", existingComment.cid);
             const authorSignerAddress = await getPKCAddressFromPublicKey(existingComment.signature.publicKey);
-            const authorDomain = getAuthorDomainFromWire(existingComment.author);
+            const authorDomain = getAuthorNameFromWire(existingComment.author);
             const authorCommunity = this._dbHandler.queryCommunityAuthor(authorSignerAddress, authorDomain);
             if (!authorCommunity) {
                 return this._publishFailedChallengeVerification({ reason: duplicateReason }, challengeRequestId);
@@ -1267,7 +1322,7 @@ export class LocalCommunity extends RpcLocalCommunity {
         if (!commentAfterAddingToIpfs)
             return undefined;
         const authorSignerAddress = await getPKCAddressFromPublicKey(commentAfterAddingToIpfs.comment.signature.publicKey);
-        const authorDomain = getAuthorDomainFromWire(commentAfterAddingToIpfs.comment.author);
+        const authorDomain = getAuthorNameFromWire(commentAfterAddingToIpfs.comment.author);
         const authorCommunity = this._dbHandler.queryCommunityAuthor(authorSignerAddress, authorDomain);
         if (!authorCommunity)
             throw Error("author.community can never be undefined after adding a comment");
@@ -1667,7 +1722,7 @@ export class LocalCommunity extends RpcLocalCommunity {
                 return messages.ERR_DUPLICATE_COMMENT_EDIT;
             const aliasSignerOfComment = this._dbHandler.queryPseudonymityAliasByCommentCid(commentToBeEdited.cid);
             if (aliasSignerOfComment) {
-                const editSignedByOriginalAuthor = commentEditPublication.signature.publicKey === aliasSignerOfComment.originalAuthorSignerPublicKey;
+                const editSignedByOriginalAuthor = commentEditPublication.signature.publicKey === aliasSignerOfComment.originalAuthorPublicKey;
                 if (!editSignedByOriginalAuthor)
                     return messages.ERR_COMMENT_EDIT_CAN_NOT_EDIT_COMMENT_IF_NOT_ORIGINAL_AUTHOR;
             }
@@ -1812,7 +1867,7 @@ export class LocalCommunity extends RpcLocalCommunity {
             return this._publishFailedChallengeVerification({ reason: messages.ERR_PUBLICATION_USES_DEPRECATED_COMMUNITY_ADDRESS }, request.challengeRequestId);
         }
         const authorSignerAddress = await getPKCAddressFromPublicKey(publication.signature.publicKey);
-        const authorDomain = getAuthorDomainFromWire(publication.author);
+        const authorDomain = getAuthorNameFromWire(publication.author);
         // Check publication props validity
         const communityAuthor = this._dbHandler.queryCommunityAuthor(authorSignerAddress, authorDomain);
         const decryptedRequestMsg = { ...request, ...decryptedRequest };
@@ -1988,8 +2043,8 @@ export class LocalCommunity extends RpcLocalCommunity {
         log.trace(`Attempting to calculate new CommentUpdate for comment (${comment.cid}) on community`, this.address);
         // This comment will have the local new CommentUpdate, which we will publish to IPFS fiels
         // It includes new author.community as well as updated values in CommentUpdate (except for replies field)
-        const storedCommentUpdate = this._dbHandler.queryStoredCommentUpdate(comment);
-        const authorDomain = getAuthorDomainFromWire(comment.author);
+        const storedCommentUpdate = this._dbHandler.queryCommentUpdateTimestampBucketReplies({ cid: comment.cid });
+        const authorDomain = getAuthorNameFromWire(comment.author);
         const calculatedCommentUpdate = this._dbHandler.queryCalculatedCommentUpdate({ comment, authorDomain });
         log.trace("Calculated comment update for comment", comment.cid, "on community", this.address, "with reply count", calculatedCommentUpdate.replyCount);
         const currentTimestamp = timestamp();
@@ -2023,7 +2078,18 @@ export class LocalCommunity extends RpcLocalCommunity {
                 };
             }
         }
-        this._addOldPageCidsToCidsToUnpin(storedCommentUpdate?.replies, commentUpdatePriorToSigning.replies).catch((err) => log.error("Failed to add old page cids of comment.replies to _cidsToUnpin", err));
+        // Extract allPageCids from the generation result (not available for singlePreloadedPage case)
+        const allPageCids = generatedRepliesPages && !("singlePreloadedPage" in generatedRepliesPages) ? generatedRepliesPages.allPageCids : undefined;
+        // Unpin old page CIDs that are no longer in the new generation
+        {
+            const oldDbReplies = storedCommentUpdate?.replies;
+            const oldCids = new Set(oldDbReplies ? Object.values(oldDbReplies).flatMap((sort) => sort?.allPageCids ?? []) : []);
+            const newCids = new Set(allPageCids ? Object.values(allPageCids).flat() : []);
+            for (const cid of oldCids) {
+                if (!newCids.has(cid))
+                    this._cidsToUnPin.add(cid);
+            }
+        }
         const newCommentUpdate = {
             ...commentUpdatePriorToSigning,
             signature: await signCommentUpdate({ update: commentUpdatePriorToSigning, signer: this.signer })
@@ -2040,12 +2106,12 @@ export class LocalCommunity extends RpcLocalCommunity {
         }
         const newCommentUpdateDbRecord = {
             ...newCommentUpdate,
+            // Store CID refs instead of full inline page data — see deriveDbReplies()
+            replies: deriveDbReplies({ replies: newCommentUpdate.replies, allPageCids }),
             postUpdatesBucket: newPostUpdateBucket,
             publishedToPostUpdatesMFS: false,
             insertedAt: timestamp()
         };
-        if (!generatedRepliesPages)
-            newCommentUpdateDbRecord.replies = undefined;
         return {
             newCommentUpdate,
             newCommentUpdateToWriteToDb: newCommentUpdateDbRecord,
@@ -2358,15 +2424,24 @@ export class LocalCommunity extends RpcLocalCommunity {
         }
     }
     async _addAllCidsUnderPurgedCommentToBeRemoved(purgedCommentAndCommentUpdate) {
-        const log = Logger("pkc-js:_addAllCidsUnderPurgedCommentToBeRemoved");
         this._cidsToUnPin.add(purgedCommentAndCommentUpdate.commentTableRow.cid);
         this._blocksToRm.push(purgedCommentAndCommentUpdate.commentTableRow.cid);
         if (typeof purgedCommentAndCommentUpdate.commentUpdateTableRow?.postUpdatesBucket === "number") {
             const localCommentUpdatePath = this._calculateLocalMfsPathForCommentUpdate(purgedCommentAndCommentUpdate.commentTableRow, purgedCommentAndCommentUpdate.commentUpdateTableRow?.postUpdatesBucket);
             this._mfsPathsToRemove.add(localCommentUpdatePath);
         }
-        if (purgedCommentAndCommentUpdate?.commentUpdateTableRow?.replies)
-            await this._addOldPageCidsToCidsToUnpin(purgedCommentAndCommentUpdate?.commentUpdateTableRow?.replies, undefined, true).catch((err) => log.error("Failed to add purged page cids to be unpinned and removed", err));
+        if (purgedCommentAndCommentUpdate?.commentUpdateTableRow?.replies) {
+            // replies is DbRepliesFormat — flat per-sort with allPageCids
+            const dbReplies = purgedCommentAndCommentUpdate.commentUpdateTableRow.replies;
+            for (const sortEntry of Object.values(dbReplies)) {
+                if (sortEntry?.allPageCids) {
+                    for (const cid of sortEntry.allPageCids) {
+                        this._cidsToUnPin.add(cid);
+                        this._blocksToRm.push(cid);
+                    }
+                }
+            }
+        }
     }
     async _purgeDisapprovedCommentsOlderThan() {
         if (typeof this.settings?.purgeDisapprovedCommentsOlderThan !== "number")
@@ -2977,12 +3052,12 @@ export class LocalCommunity extends RpcLocalCommunity {
             await this.initDbHandlerIfNeeded();
             await this._dbHandler.initDbIfNeeded();
             const cidsAndReplies = this._dbHandler.queryAllCommentCidsAndTheirReplies();
-            cidsAndReplies.forEach((comment) => this._cidsToUnPin.add(comment.cid));
-            await Promise.all(cidsAndReplies
-                .filter((comment) => comment.replies)
-                .map(async (commentWithReplies) => {
-                await this._addOldPageCidsToCidsToUnpin(commentWithReplies.replies, undefined);
-            }));
+            for (const comment of cidsAndReplies) {
+                this._cidsToUnPin.add(comment.cid);
+                for (const pageCid of comment.allPageCids) {
+                    this._cidsToUnPin.add(pageCid);
+                }
+            }
         }
         catch (e) {
             log.error("Failed to query all cids under this community to delete them", e);