@atproto/bsky 0.3.12 → 0.4.0

package/dist/hydration/hydrator.js

@@ -6,12 +6,11 @@ import { NOTIFICATION_REASON } from '../api/app/bsky/notification/constants.js';
 import { app, chat, com, place, site, social } from '../lexicons/index.js';
 import { hydrationLogger } from '../logger.js';
 import { events } from '../telemetry/events.js';
-import { SITE_STANDARD_NSID_PREFIX } from '../util/external/standard-site.js';
-import { uriToDid, uriToDid as didFromUri } from '../util/uris.js';
+import { recordUriOf, uriToDid, uriToDid as didFromUri } from '../util/uris.js';
 import { MAX_PROFILE_LINKS } from '../views/profile-link-rules.js';
-import { isExternalEmbedType, isListRuleType, isRecordEmbedType, isRecordWithMediaType, } from '../views/types.js';
+import { isListRuleType, isRecordEmbedType, isRecordWithMediaType, } from '../views/types.js';
 import { ActorHydrator, } from './actor.js';
-import { ExternalHydrator, actionExternalRecordTakedowns, genericRecordKey, parseGenericRecordKey, trackExternalHydration, } from './external.js';
+import { EXTERNAL_HYDRATION_LIMITS, ExternalHydrator, MAX_CONCURRENT_BACKLINK_REQUESTS, MAX_EXTERNAL_HYDRATION_PASSES, actionExternalRecordTakedowns, externalAssociatedRefsFromPosts, genericRecordKey, trackExternalHydration, } from './external.js';
 import { FeedHydrator, } from './feed.js';
 import { GraphHydrator, } from './graph.js';
 import { LabelHydrator, Labels, } from './label.js';
@@ -424,13 +423,7 @@ export class Hydrator {
                 postUrisWithPostgates.add(uri);
             }
         }
-        const siteStandardRefs = siteStandardRefsFromPosts(postsLayer0, postsLayer1, postsLayer2);
-        const siteStandardLabelSubjects = dedupeStrs(siteStandardRefs.map((ref) => ref.uri));
-        // Site-standard record owners need profile basics so we can populate
-        // `associatedProfiles` on the external embed view. Include them in the
-        // post-author profile fetch below; any DIDs surfaced later by the
-        // dataplane are picked up by the top-up after the parallel batch.
-        const ssRefDids = siteStandardRefs.map((ref) => uriToDid(ref.uri));
+        const externalRefs = externalAssociatedRefsFromPosts(postsLayer0, postsLayer1, postsLayer2);
         // Collect known-liker profiles and author-liker block relationships.
         const knownLikers = await knownLikersPromise;
         const knownLikerDids = [];
@@ -446,15 +439,14 @@ export class Hydrator {
         });
         const knownProfileDids = dedupeStrs([
             ...allPostUris.map(didFromUri),
-            ...ssRefDids,
             ...knownLikerDids,
         ]);
-        const [postAggs, postViewers, labels, { postBlocks, additionalRelationshipsBidirectionalBlocks: authorsAndKnownLikersBidirectionalBlocks, }, profileState, listState, feedGenState, labelerState, starterPackState, postgates, { documents: siteStandardDocuments, publications: siteStandardPublications, },] = await Promise.all([
+        const [postAggs, postViewers, labels, { postBlocks, additionalRelationshipsBidirectionalBlocks: authorsAndKnownLikersBidirectionalBlocks, }, profileState, listState, feedGenState, labelerState, starterPackState, postgates, externalState,] = await Promise.all([
             this.feed.getPostAggregates(allRefs, ctx.viewer),
             ctx.viewer
                 ? this.feed.getPostViewerStates(threadRefs, ctx.viewer)
                 : undefined,
-            this.label.getLabelsForSubjects([...allPostUris, ...siteStandardLabelSubjects], ctx.labelers),
+            this.label.getLabelsForSubjects(allPostUris, ctx.labelers),
             this.hydratePostBlocks(posts, ctx, authorsAndKnownLikersRelationships),
             this.hydrateProfiles(knownProfileDids, ctx),
             this.hydrateLists([...nestedListUris, ...threadgateListUris], ctx),
@@ -462,29 +454,15 @@ export class Hydrator {
             this.hydrateLabelers(nestedLabelerDids, ctx),
             this.hydrateStarterPacksBasic(nestedStarterPackUris, ctx),
             this.feed.getPostgatesForPosts([...postUrisWithPostgates.values()]),
-            this.external.getSiteStandardRecordsByRef(siteStandardRefs, ctx.includeTakedowns),
+            this.hydrateEmbedExternalViewFromRefs(externalRefs, ctx, {
+                hydration: 'basic',
+            }),
         ]);
         if (!ctx.includeTakedowns) {
             actionTakedownLabels(allPostUris, posts, labels);
-            actionSiteStandardTakedownLabels(siteStandardDocuments, siteStandardPublications, labels);
-        }
-        // Defensive top-up: in the unlikely case the dataplane returned a
-        // publication owned by a DID not present in the post-author or
-        // pinned-ref sets, fetch its profile serially and fold it in.
-        const knownProfileDidsSet = new Set(knownProfileDids);
-        const extraSsDids = [];
-        for (const key of siteStandardPublications.keys()) {
-            const did = uriToDid(parseGenericRecordKey(key).uri);
-            if (!knownProfileDidsSet.has(did)) {
-                knownProfileDidsSet.add(did);
-                extraSsDids.push(did);
-            }
         }
-        const mergedProfileState = extraSsDids.length
-            ? mergeStates(profileState, await this.hydrateProfilesBasic(extraSsDids, ctx))
-            : profileState;
         // combine all hydration state
-        return mergeManyStates(mergedProfileState, listState, feedGenState, labelerState, starterPackState, {
+        return mergeManyStates(externalState, profileState, listState, feedGenState, labelerState, starterPackState, {
             posts,
             postAggs,
             postViewers,
@@ -494,8 +472,6 @@ export class Hydrator {
             labels,
             threadgates,
             postgates,
-            siteStandardDocuments,
-            siteStandardPublications,
             ctx,
         });
     }
@@ -626,20 +602,23 @@ export class Hydrator {
      * generic batch within the traversal (the root batch is pass 1); nested
      * batches continue the count. Nested passes share the caller's
      * `hydrationMetrics` tracker; without one, this call owns the traversal and
-     * reports its totals.
+     * reports its totals. `options.hydration` (default `'detailed'`) applies to
+     * the whole traversal and is propagated to every nested pass.
      */
-    async hydrateEmbedExternalViewFromUris(uris, ctx, seenRecordKeys = new Set(), hydrationPass = 1, hydrationMetrics) {
-        return trackExternalHydration('uris', ctx.xrpcEndpoint, hydrationMetrics, (hydrationMetrics) => this.hydrateEmbedExternalView(uris.map(uriToRef), ctx, seenRecordKeys, hydrationPass, hydrationMetrics));
+    async hydrateEmbedExternalViewFromUris(uris, ctx, options = {}) {
+        const { hydration = 'detailed', seenRecordKeys = new Set(), hydrationPass = 1, hydrationMetrics, } = options;
+        return trackExternalHydration('uris', ctx.xrpcEndpoint, hydrationMetrics, (hydrationMetrics) => this.hydrateEmbedExternalView(uris.map(uriToRef), ctx, hydration, seenRecordKeys, hydrationPass, hydrationMetrics));
     }
     /**
      * Like `hydrateEmbedExternalViewFromUris`, but for exact record versions
      * pinned by strong refs. Roots land only in `externalRecordsByRef`; their
      * URI dependencies are hydrated as latest versions by the URI path.
      */
-    async hydrateEmbedExternalViewFromRefs(refs, ctx, seenRecordKeys = new Set(), hydrationPass = 1, hydrationMetrics) {
-        return trackExternalHydration('refs', ctx.xrpcEndpoint, hydrationMetrics, (hydrationMetrics) => this.hydrateEmbedExternalView(refs, ctx, seenRecordKeys, hydrationPass, hydrationMetrics));
+    async hydrateEmbedExternalViewFromRefs(refs, ctx, options = {}) {
+        const { hydration = 'detailed', seenRecordKeys = new Set(), hydrationPass = 1, hydrationMetrics, } = options;
+        return trackExternalHydration('refs', ctx.xrpcEndpoint, hydrationMetrics, (hydrationMetrics) => this.hydrateEmbedExternalView(refs, ctx, hydration, seenRecordKeys, hydrationPass, hydrationMetrics));
     }
-    async hydrateEmbedExternalView(refs, ctx, seenRecordKeys, hydrationPass, hydrationMetrics) {
+    async hydrateEmbedExternalView(refs, ctx, hydration, seenRecordKeys, hydrationPass, hydrationMetrics) {
         // Filter at the entry point so each lookup runs at most once per shared
         // traversal, even for callers that did not prefilter their input. Null or
         // unavailable results stay seen as completed lookups. Nothing unseen means
@@ -699,7 +678,12 @@ export class Hydrator {
         });
         actionExternalRecordTakedowns(baseState, ctx.includeTakedowns);
         // Discover and hydrate any additional view dependencies
-        const depsState = await this.hydrateExternalViewDependencies(baseState, items, ctx, seenRecordKeys, hydrationPass, hydrationMetrics);
+        const depsState = await this.hydrateExternalViewDependencies(baseState, items, ctx, {
+            hydration,
+            seenRecordKeys,
+            hydrationPass,
+            hydrationMetrics,
+        });
         // A supplemental latest lookup returning the same URI and CID is the same
         // version, so it may replace an unavailable exact entry; other CIDs have
         // other keys and never satisfy a pin.
@@ -718,13 +702,18 @@ export class Hydrator {
      * `hydrationPass` is the number of the generic batch that produced `state`
      * (1-based, default 1 for prehydrated state, which is treated as the root
      * batch); the next nested batch is `hydrationPass + 1`. `hydrationMetrics`
-     * is shared as in `hydrateEmbedExternalViewFromUris`.
+     * is shared as in `hydrateEmbedExternalViewFromUris`. `options.hydration`
+     * (default `'detailed'`) decides, per collection, whether social-preview
+     * samples are scheduled; backlink counts and content dependencies are
+     * discovered at both levels.
      */
-    async hydrateExternalViewDependencies(state, refs, ctx, seenRecordKeys = new Set(), hydrationPass = 1, hydrationMetrics) {
-        return trackExternalHydration('dependencies', ctx.xrpcEndpoint, hydrationMetrics, (hydrationMetrics) => this.discoverExternalViewDependencies(state, refs, ctx, seenRecordKeys, hydrationPass, hydrationMetrics));
+    async hydrateExternalViewDependencies(state, refs, ctx, options = {}) {
+        const { hydration = 'detailed', seenRecordKeys = new Set(), hydrationPass = 1, hydrationMetrics, } = options;
+        return trackExternalHydration('dependencies', ctx.xrpcEndpoint, hydrationMetrics, (hydrationMetrics) => this.discoverExternalViewDependencies(state, refs, ctx, hydration, seenRecordKeys, hydrationPass, hydrationMetrics));
     }
-    async discoverExternalViewDependencies(state, refs, ctx, seenRecordKeys, hydrationPass, hydrationMetrics) {
-        // Backlink sources to sample per target, each with its own bound.
+    async discoverExternalViewDependencies(state, refs, ctx, hydration, seenRecordKeys, hydrationPass, hydrationMetrics) {
+        // Social-preview backlink sources to sample per target, each with its own
+        // bound. A target with no samples still gets its aggregate count.
         const targets = new Map();
         const discoveredUris = new Set();
         const rootBatch = !hydrationMetrics.rootsCounted;
@@ -756,7 +745,8 @@ export class Hydrator {
             if (!parsed.success ||
                 !isDidString(parsed.value.authority) ||
                 !parsed.value.rkey ||
-                parsed.value.hash) {
+                parsed.value.query !== undefined ||
+                parsed.value.hash !== undefined) {
                 continue;
             }
             /*
@@ -765,24 +755,30 @@ export class Hydrator {
             const { collection } = parsed.value;
             if (collection === site.standard.document.$type &&
                 site.standard.document.$matches(record, { strict: false })) {
-                targets.set(uri, [
-                    {
-                        collection: site.standard.graph.recommend.$type,
-                        limit: ExternalHydrator.MAX_BACKLINK_PREVIEWS,
-                    },
-                ]);
+                const limits = EXTERNAL_HYDRATION_LIMITS[site.standard.document.$type];
+                targets.set(uri, hydration === 'detailed'
+                    ? [
+                        {
+                            collection: site.standard.graph.recommend.$type,
+                            limit: limits.recommendationPreviews,
+                        },
+                    ]
+                    : []);
                 const publicationUri = recordUriOf(record.site, site.standard.publication.$type);
                 if (publicationUri)
                     discoveredUris.add(publicationUri);
             }
             else if (collection === site.standard.publication.$type &&
                 site.standard.publication.$matches(record, { strict: false })) {
-                targets.set(uri, [
-                    {
-                        collection: site.standard.graph.subscription.$type,
-                        limit: ExternalHydrator.MAX_BACKLINK_PREVIEWS,
-                    },
-                ]);
+                const limits = EXTERNAL_HYDRATION_LIMITS[site.standard.publication.$type];
+                targets.set(uri, hydration === 'detailed'
+                    ? [
+                        {
+                            collection: site.standard.graph.subscription.$type,
+                            limit: limits.subscriberPreviews,
+                        },
+                    ]
+                    : []);
             }
             else if (collection === place.stream.livestream.$type &&
                 place.stream.livestream.$matches(record, { strict: false })) {
@@ -790,16 +786,20 @@ export class Hydrator {
             }
             else if (collection === social.grain.gallery.$type &&
                 social.grain.gallery.$matches(record, { strict: false })) {
-                targets.set(uri, [
-                    {
+                const limits = EXTERNAL_HYDRATION_LIMITS[social.grain.gallery.$type];
+                // Membership items are content, sampled at every level.
+                const samples = [];
+                if (hydration === 'detailed') {
+                    samples.push({
                         collection: social.grain.favorite.$type,
-                        limit: ExternalHydrator.MAX_BACKLINK_PREVIEWS,
-                    },
-                    {
-                        collection: social.grain.gallery.item.$type,
-                        limit: ExternalHydrator.MAX_BACKLINK_GALLERY_ITEMS,
-                    },
-                ]);
+                        limit: limits.favoritePreviews,
+                    });
+                }
+                samples.push({
+                    collection: social.grain.gallery.item.$type,
+                    limit: limits.items,
+                });
+                targets.set(uri, samples);
             }
             else if (collection === social.grain.gallery.item.$type &&
                 social.grain.gallery.item.$matches(record, { strict: false })) {
@@ -826,7 +826,7 @@ export class Hydrator {
                 }),
                 (async () => {
                     // @NOTE Bound fan-out as well as each sample; counts are independent.
-                    for (const batch of chunkArray(queries, ExternalHydrator.MAX_BACKLINK_FANOUT)) {
+                    for (const batch of chunkArray(queries, MAX_CONCURRENT_BACKLINK_REQUESTS)) {
                         await Promise.all(batch.map(async ({ uri, collection, limit }) => {
                             let page;
                             try {
@@ -838,9 +838,11 @@ export class Hydrator {
                                 return;
                             }
                             const links = page.backlinks
-                                .filter((b) => parseAtUriString(b.uri).success)
-                                .slice(0, limit)
-                                .map((b) => b.uri);
+                                .flatMap(({ uri }) => {
+                                const link = recordUriOf(uri, collection);
+                                return link ? [link] : [];
+                            })
+                                .slice(0, limit);
                             backlinks.set(uri, dedupeStrs([...(backlinks.get(uri) ?? []), ...links]));
                             links.forEach((link) => discoveredUris.add(link));
                         }));
@@ -852,20 +854,24 @@ export class Hydrator {
         }
         const newUris = [...discoveredUris].filter((uri) => !seenRecordKeys.has(genericRecordKey({ uri })));
         let nested = {};
-        if (newUris.length &&
-            hydrationPass >= ExternalHydrator.MAX_EXTERNAL_HYDRATION_PASSES) {
+        if (newUris.length && hydrationPass >= MAX_EXTERNAL_HYDRATION_PASSES) {
             // Keep the partial state. Skipped URIs are not marked seen: they were
             // never fetched.
             hydrationMetrics.capped = true;
             hydrationLogger.warn({
                 hydrationPass,
-                maxPasses: ExternalHydrator.MAX_EXTERNAL_HYDRATION_PASSES,
+                maxPasses: MAX_EXTERNAL_HYDRATION_PASSES,
                 skipped: newUris.length,
             }, 'external hydration pass limit reached, skipping dependencies');
         }
         else if (newUris.length) {
             try {
-                nested = await this.hydrateEmbedExternalViewFromUris(newUris, ctx, seenRecordKeys, hydrationPass + 1, hydrationMetrics);
+                nested = await this.hydrateEmbedExternalViewFromUris(newUris, ctx, {
+                    hydration,
+                    seenRecordKeys,
+                    hydrationPass: hydrationPass + 1,
+                    hydrationMetrics,
+                });
             }
             catch (err) {
                 // Skip the whole batch; partial child state could miss moderation.
@@ -1573,52 +1579,6 @@ const nestedRecordUris = (post) => {
     }
     return uris;
 };
-/**
- * Returns every strongRef the post embedded as `external.associatedRefs`,
- * regardless of collection. Callers are responsible for filtering to NSIDs
- * they care about.
- */
-const externalAssociatedRefs = (post) => {
-    const embed = post?.embed;
-    if (!embed)
-        return [];
-    if (isExternalEmbedType(embed)) {
-        return embed.external.associatedRefs ?? [];
-    }
-    if (isRecordWithMediaType(embed) && isExternalEmbedType(embed.media)) {
-        return embed.media.external.associatedRefs ?? [];
-    }
-    return [];
-};
-/**
- * Collects standard.site refs across all post layers, deduped by `uri+cid` so
- * the dataplane batch is minimal even if multiple posts in the layer reference
- * the same exact version of an SS record. The same `${uri}@${cid}` keys are
- * what `getSiteStandardRecordsByRef` uses on the resulting hydration maps,
- * which lets `lookupAssociatedSiteStandardRecords` do an O(1) version-exact
- * lookup at view time.
- */
-const siteStandardRefsFromPosts = (...postLayers) => {
-    const seen = new Set();
-    const out = [];
-    for (const layer of postLayers) {
-        for (const item of layer.values()) {
-            if (!item)
-                continue;
-            for (const ref of externalAssociatedRefs(item.record)) {
-                if (!new AtUri(ref.uri).collection.startsWith(SITE_STANDARD_NSID_PREFIX)) {
-                    continue;
-                }
-                const key = `${ref.uri}@${ref.cid}`;
-                if (seen.has(key))
-                    continue;
-                seen.add(key);
-                out.push({ uri: ref.uri, cid: ref.cid });
-            }
-        }
-    }
-    return out;
-};
 const getListUrisFromThreadgates = (gates) => {
     const uris = [];
     for (const gate of gates.values()) {
@@ -1708,8 +1668,6 @@ export const mergeStates = (stateA, stateB) => {
         externalRecordsByRef: mergeMaps(stateA.externalRecordsByRef, stateB.externalRecordsByRef),
         externalRecordBacklinks: mergeMaps(stateA.externalRecordBacklinks, stateB.externalRecordBacklinks),
         externalRecordBacklinkCounts: mergeMaps(stateA.externalRecordBacklinkCounts, stateB.externalRecordBacklinkCounts),
-        siteStandardDocuments: mergeMaps(stateA.siteStandardDocuments, stateB.siteStandardDocuments),
-        siteStandardPublications: mergeMaps(stateA.siteStandardPublications, stateB.siteStandardPublications),
     };
 };
 export const mergeManyStates = (...states) => {
@@ -1722,71 +1680,6 @@ const actionTakedownLabels = (keys, hydrationMap, labels) => {
         }
     }
 };
-/**
- * The URI when an externally authored reference is a complete DID-based
- * record URI in `collection`, with a record key and no fragment.
- */
-const recordUriOf = (value, collection) => {
-    const parsed = parseAtUriString(value);
-    return parsed.success &&
-        isDidString(parsed.value.authority) &&
-        parsed.value.collection === collection &&
-        parsed.value.rkey &&
-        !parsed.value.hash
-        ? value
-        : undefined;
-};
-/**
- * Apply takedown labels to the site.standard hydration maps. Per-record
- * takedowns null any entry whose subject URI is taken down; pair takedowns
- * propagate that null across doc/publication pairs so we never render half
- * of a moderated embed. Pairs are discovered via `doc.record.site`; orphan
- * docs and orphan publications are subject only to the per-record sweep.
- */
-const actionSiteStandardTakedownLabels = (documents, publications, labels) => {
-    // Pairings have to be captured before nulling — the doc record carries
-    // the publication URI in its `site` field, and we lose it once we null.
-    const pairings = [];
-    if (documents.size > 0 && publications.size > 0) {
-        const pubKeysByUri = new Map();
-        for (const key of publications.keys()) {
-            const { uri } = parseGenericRecordKey(key);
-            const list = pubKeysByUri.get(uri);
-            if (list)
-                list.push(key);
-            else
-                pubKeysByUri.set(uri, [key]);
-        }
-        for (const [docKey, doc] of documents) {
-            const site = doc?.record.site;
-            if (!site || !site.startsWith('at://'))
-                continue;
-            for (const pubKey of pubKeysByUri.get(site) ?? []) {
-                pairings.push({ docKey, pubKey });
-            }
-        }
-    }
-    // Per-record takedowns: null any entry whose subject URI is taken down.
-    for (const key of documents.keys()) {
-        const { uri } = parseGenericRecordKey(key);
-        if (labels.get(uri)?.isTakendown)
-            documents.set(key, null);
-    }
-    for (const key of publications.keys()) {
-        const { uri } = parseGenericRecordKey(key);
-        if (labels.get(uri)?.isTakendown)
-            publications.set(key, null);
-    }
-    // Pair takedowns: if exactly one side of a pair was nulled above, mirror.
-    for (const { docKey, pubKey } of pairings) {
-        const doc = documents.get(docKey);
-        const pub = publications.get(pubKey);
-        if (doc === null && pub !== null)
-            publications.set(pubKey, null);
-        if (pub === null && doc !== null)
-            documents.set(docKey, null);
-    }
-};
 const uriToRef = (uri) => {
     return { uri };
 };