@atproto/bsky 0.0.232 → 0.0.234

package/src/hydration/hydrator.ts

@@ -1,5 +1,5 @@
 import assert from 'node:assert'
-import { mapDefined } from '@atproto/common'
+import { dedupeStrs, mapDefined } from '@atproto/common'
 import { AtUri, AtUriString, DidString, UriString } from '@atproto/syntax'
 import { DataPlaneClient } from '../data-plane/client/index.js'
 import {
@@ -14,10 +14,15 @@ import {
   Notification,
   RecordRef,
 } from '../proto/bsky_pb.js'
+import {
+  SITE_STANDARD_NSID_PREFIX,
+  parseSiteStandardRecordKey,
+} from '../util/standard-site.js'
 import { uriToDid, uriToDid as didFromUri } from '../util/uris.js'
 import { ParsedLabelers } from '../util.js'
 import {
   ProfileRecord,
+  isExternalEmbedType,
   isListRuleType,
   isRecordEmbedType,
   isRecordWithMediaType,
@@ -31,6 +36,11 @@ import {
   ProfileViewerState,
   ProfileViewerStates,
 } from './actor.js'
+import {
+  ExternalHydrator,
+  SiteStandardDocuments,
+  SiteStandardPublications,
+} from './external.js'
 import {
   FeedGenAggs,
   FeedGenViewerStates,
@@ -151,6 +161,8 @@ export type HydrationState = {
   bidirectionalBlocks?: BidirectionalBlocks
   verifications?: Verifications
   bookmarks?: Bookmarks
+  siteStandardDocuments?: SiteStandardDocuments
+  siteStandardPublications?: SiteStandardPublications
 }
 
 export type PostBlock = { embed: boolean; parent: boolean; root: boolean }
@@ -193,6 +205,7 @@ export type HydratorConfig = {
 
 export class Hydrator {
   actor: ActorHydrator
+  external: ExternalHydrator
   feed: FeedHydrator
   graph: GraphHydrator
   label: LabelHydrator
@@ -206,6 +219,7 @@ export class Hydrator {
   ) {
     this.config = config
     this.actor = new ActorHydrator(dataplane)
+    this.external = new ExternalHydrator(dataplane)
     this.feed = new FeedHydrator(dataplane)
     this.graph = new GraphHydrator(dataplane)
     this.label = new LabelHydrator(dataplane)
@@ -590,6 +604,24 @@ export class Hydrator {
       }
     }
 
+    const siteStandardRefs = siteStandardRefsFromPosts(
+      postsLayer0,
+      postsLayer1,
+      postsLayer2,
+    )
+    const siteStandardLabelSubjects = dedupeStrs(
+      siteStandardRefs.map((ref) => ref.uri),
+    )
+    // Site-standard record owners need profile basics so we can populate
+    // `associatedProfiles` on the external embed view. Include them in the
+    // post-author profile fetch below; any DIDs surfaced later by the
+    // dataplane are picked up by the top-up after the parallel batch.
+    const ssRefDids = siteStandardRefs.map((ref) => uriToDid(ref.uri))
+    const knownProfileDids = dedupeStrs([
+      ...allPostUris.map(didFromUri),
+      ...ssRefDids,
+    ])
+
     const [
       postAggs,
       postViewers,
@@ -601,26 +633,61 @@ export class Hydrator {
       labelerState,
       starterPackState,
       postgates,
+      {
+        documents: siteStandardDocuments,
+        publications: siteStandardPublications,
+      },
     ] = await Promise.all([
       this.feed.getPostAggregates(allRefs, ctx.viewer),
       ctx.viewer
         ? this.feed.getPostViewerStates(threadRefs, ctx.viewer)
         : undefined,
-      this.label.getLabelsForSubjects(allPostUris, ctx.labelers),
+      this.label.getLabelsForSubjects(
+        [...allPostUris, ...siteStandardLabelSubjects],
+        ctx.labelers,
+      ),
       this.hydratePostBlocks(posts, ctx),
-      this.hydrateProfiles(allPostUris.map(didFromUri), ctx),
+      this.hydrateProfiles(knownProfileDids, ctx),
       this.hydrateLists([...nestedListUris, ...threadgateListUris], ctx),
       this.hydrateFeedGens(nestedFeedGenUris, ctx),
       this.hydrateLabelers(nestedLabelerDids, ctx),
       this.hydrateStarterPacksBasic(nestedStarterPackUris, ctx),
       this.feed.getPostgatesForPosts([...postUrisWithPostgates.values()]),
+      this.external.getSiteStandardRecordsByRef(
+        siteStandardRefs,
+        ctx.includeTakedowns,
+      ),
     ])
     if (!ctx.includeTakedowns) {
       actionTakedownLabels(allPostUris, posts, labels)
+      actionSiteStandardTakedownLabels(
+        siteStandardDocuments,
+        siteStandardPublications,
+        labels,
+      )
+    }
+
+    // Defensive top-up: in the unlikely case the dataplane returned a
+    // publication owned by a DID not present in the post-author or
+    // pinned-ref sets, fetch its profile serially and fold it in.
+    const knownProfileDidsSet = new Set(knownProfileDids)
+    const extraSsDids: DidString[] = []
+    for (const key of siteStandardPublications.keys()) {
+      const did = uriToDid(parseSiteStandardRecordKey(key).uri)
+      if (!knownProfileDidsSet.has(did)) {
+        knownProfileDidsSet.add(did)
+        extraSsDids.push(did)
+      }
     }
+    const mergedProfileState = extraSsDids.length
+      ? mergeStates(
+          profileState,
+          await this.hydrateProfilesBasic(extraSsDids, ctx),
+        )
+      : profileState
     // combine all hydration state
     return mergeManyStates(
-      profileState,
+      mergedProfileState,
       listState,
       feedGenState,
       labelerState,
@@ -633,6 +700,8 @@ export class Hydrator {
         labels,
         threadgates,
         postgates,
+        siteStandardDocuments,
+        siteStandardPublications,
         ctx,
       },
     )
@@ -753,6 +822,58 @@ export class Hydrator {
     })
   }
 
+  /**
+   * Hydrate the state needed to build an `app.bsky.embed.external#view` for
+   * a set of `site.standard.*` AT-URIs at compose-time. Filters input URIs to
+   * SS collections, fetches latest indexed versions plus labels, and gates
+   * taken-down records.
+   */
+  async hydrateEmbedExternalViewFromUris(
+    uris: AtUriString[],
+    ctx: HydrateCtx,
+  ): Promise<HydrationState> {
+    const ssUris = dedupeStrs(
+      uris.filter((u) =>
+        new AtUri(u).collection.startsWith(SITE_STANDARD_NSID_PREFIX),
+      ),
+    ) as AtUriString[]
+    if (!ssUris.length) return { ctx }
+    const dids = dedupeStrs(ssUris.map((uri) => uriToDid(uri)))
+
+    const [{ documents, publications }, labels, profiles] = await Promise.all([
+      this.external.getSiteStandardRecordsByURI(ssUris, ctx.includeTakedowns),
+      this.label.getLabelsForSubjects(ssUris, ctx.labelers),
+      this.hydrateProfilesBasic(dids, ctx),
+    ])
+    if (!ctx.includeTakedowns) {
+      actionSiteStandardTakedownLabels(documents, publications, labels)
+    }
+    // Edge case: a document's `site` may resolve to a publication owned by a
+    // different repo than any of the input URIs (the dataplane returns it
+    // even though it wasn't requested directly). Top up profile coverage for
+    // any such DIDs with a serial second hydration so `associatedProfiles`
+    // is complete.
+    const knownDids = new Set<string>(dids)
+    const extraDids: DidString[] = []
+    for (const key of publications.keys()) {
+      const did = uriToDid(parseSiteStandardRecordKey(key).uri)
+      if (!knownDids.has(did)) {
+        knownDids.add(did)
+        extraDids.push(did)
+      }
+    }
+    const profilesState = extraDids.length
+      ? mergeStates(profiles, await this.hydrateProfilesBasic(extraDids, ctx))
+      : profiles
+
+    return mergeStates(profilesState, {
+      ctx,
+      labels,
+      siteStandardDocuments: documents,
+      siteStandardPublications: publications,
+    })
+  }
+
   // app.bsky.feed.defs#threadViewPost
   // - post
   //   - profile
@@ -1480,6 +1601,55 @@ const nestedRecordUris = (post: Post['record']): AtUriString[] => {
   return uris
 }
 
+/**
+ * Returns every strongRef the post embedded as `external.associatedRefs`,
+ * regardless of collection. Callers are responsible for filtering to NSIDs
+ * they care about.
+ */
+const externalAssociatedRefs = (
+  post: Post['record'],
+): { uri: AtUriString; cid: string }[] => {
+  const embed = post?.embed
+  if (!embed) return []
+  if (isExternalEmbedType(embed)) {
+    return embed.external.associatedRefs ?? []
+  }
+  if (isRecordWithMediaType(embed) && isExternalEmbedType(embed.media)) {
+    return embed.media.external.associatedRefs ?? []
+  }
+  return []
+}
+
+/**
+ * Collects standard.site refs across all post layers, deduped by `uri+cid` so
+ * the dataplane batch is minimal even if multiple posts in the layer reference
+ * the same exact version of an SS record. The same `${uri}@${cid}` keys are
+ * what `getSiteStandardRecordsByRef` uses on the resulting hydration maps,
+ * which lets `lookupAssociatedSiteStandardRecords` do an O(1) version-exact
+ * lookup at view time.
+ */
+const siteStandardRefsFromPosts = (...postLayers: Posts[]): ItemRef[] => {
+  const seen = new Set<string>()
+  const out: ItemRef[] = []
+  for (const layer of postLayers) {
+    for (const item of layer.values()) {
+      if (!item) continue
+      for (const ref of externalAssociatedRefs(item.record)) {
+        if (
+          !new AtUri(ref.uri).collection.startsWith(SITE_STANDARD_NSID_PREFIX)
+        ) {
+          continue
+        }
+        const key = `${ref.uri}@${ref.cid}`
+        if (seen.has(key)) continue
+        seen.add(key)
+        out.push({ uri: ref.uri, cid: ref.cid })
+      }
+    }
+  }
+  return out
+}
+
 const getListUrisFromThreadgates = (gates: Threadgates): AtUriString[] => {
   const uris: AtUriString[] = []
   for (const gate of gates.values()) {
@@ -1560,6 +1730,14 @@ export const mergeStates = (
     ),
     verifications: mergeMaps(stateA.verifications, stateB.verifications),
     bookmarks: mergeNestedMaps(stateA.bookmarks, stateB.bookmarks),
+    siteStandardDocuments: mergeMaps(
+      stateA.siteStandardDocuments,
+      stateB.siteStandardDocuments,
+    ),
+    siteStandardPublications: mergeMaps(
+      stateA.siteStandardPublications,
+      stateB.siteStandardPublications,
+    ),
   }
 }
 
@@ -1579,6 +1757,57 @@ const actionTakedownLabels = (
   }
 }
 
+/**
+ * Apply takedown labels to the site.standard hydration maps. Per-record
+ * takedowns null any entry whose subject URI is taken down; pair takedowns
+ * propagate that null across doc/publication pairs so we never render half
+ * of a moderated embed. Pairs are discovered via `doc.record.site`; orphan
+ * docs and orphan publications are subject only to the per-record sweep.
+ */
+const actionSiteStandardTakedownLabels = (
+  documents: SiteStandardDocuments,
+  publications: SiteStandardPublications,
+  labels: Labels,
+) => {
+  // Pairings have to be captured before nulling — the doc record carries
+  // the publication URI in its `site` field, and we lose it once we null.
+  const pairings: { docKey: string; pubKey: string }[] = []
+  if (documents.size > 0 && publications.size > 0) {
+    const pubKeysByUri = new Map<string, string[]>()
+    for (const key of publications.keys()) {
+      const { uri } = parseSiteStandardRecordKey(key)
+      const list = pubKeysByUri.get(uri)
+      if (list) list.push(key)
+      else pubKeysByUri.set(uri, [key])
+    }
+    for (const [docKey, doc] of documents) {
+      const site = doc?.record.site
+      if (!site || !site.startsWith('at://')) continue
+      for (const pubKey of pubKeysByUri.get(site) ?? []) {
+        pairings.push({ docKey, pubKey })
+      }
+    }
+  }
+
+  // Per-record takedowns: null any entry whose subject URI is taken down.
+  for (const key of documents.keys()) {
+    const { uri } = parseSiteStandardRecordKey(key)
+    if (labels.get(uri)?.isTakendown) documents.set(key, null)
+  }
+  for (const key of publications.keys()) {
+    const { uri } = parseSiteStandardRecordKey(key)
+    if (labels.get(uri)?.isTakendown) publications.set(key, null)
+  }
+
+  // Pair takedowns: if exactly one side of a pair was nulled above, mirror.
+  for (const { docKey, pubKey } of pairings) {
+    const doc = documents.get(docKey)
+    const pub = publications.get(pubKey)
+    if (doc === null && pub !== null) publications.set(pubKey, null)
+    if (pub === null && doc !== null) documents.set(docKey, null)
+  }
+}
+
 const uriToRef = (uri: AtUriString): ItemRef => {
   return { uri }
 }

package/src/logger.ts

@@ -19,6 +19,8 @@ export const dataplaneLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('bsky:dp')
 export const ageAssuranceLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('bsky:aa')
+export const viewsLogger: ReturnType<typeof subsystemLogger> =
+  subsystemLogger('bsky:views')
 export const httpLogger: ReturnType<typeof subsystemLogger> =
   subsystemLogger('bsky')
 

package/src/util/standard-site.test.ts

@@ -0,0 +1,271 @@
+import { describe, expect, it } from 'vitest'
+import { validateStandardSiteForUrl } from './standard-site.js'
+
+const docDid = 'did:plc:doc'
+const pubDid = 'did:plc:pub'
+const docUri = `at://${docDid}/site.standard.document/abc`
+const pubUri = `at://${pubDid}/site.standard.publication/self`
+
+const makeDoc = (record: { site: string; path?: string }) => ({
+  ref: { uri: docUri },
+  info: { record },
+})
+
+const makePub = (record: { url: string }) => ({
+  ref: { uri: pubUri },
+  info: { record },
+})
+
+describe(validateStandardSiteForUrl, () => {
+  describe('document + publication', () => {
+    it('accepts when site resolves to publication and url+path matches', () => {
+      const doc = makeDoc({ site: pubUri, path: '/posts/hello' })
+      const pub = makePub({ url: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(doc, pub, 'https://example.com/posts/hello'),
+      ).toBe(true)
+    })
+
+    it('rejects when joined url+path does not match assumedUrl', () => {
+      const doc = makeDoc({ site: pubUri, path: '/posts/hello' })
+      const pub = makePub({ url: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(
+          doc,
+          pub,
+          'https://example.com/posts/different',
+        ),
+      ).toBe(false)
+    })
+
+    it('accepts when document has no path and assumedUrl is the bare publication url', () => {
+      const doc = makeDoc({ site: pubUri })
+      const pub = makePub({ url: 'https://example.com' })
+      expect(validateStandardSiteForUrl(doc, pub, 'https://example.com')).toBe(
+        true,
+      )
+    })
+  })
+
+  describe('document only (loose doc)', () => {
+    it('accepts when site is HTTP and matches assumedUrl', () => {
+      const doc = makeDoc({ site: 'https://example.com', path: '/posts/hi' })
+      expect(
+        validateStandardSiteForUrl(
+          doc,
+          undefined,
+          'https://example.com/posts/hi',
+        ),
+      ).toBe(true)
+    })
+
+    // The lookups reject this pairing upstream, but validate as a fallback:
+    // an at-uri base canonicalizes to null and fails the URL match.
+    it('rejects when site is an at-uri (non-HTTP base canonicalizes to null)', () => {
+      const doc = makeDoc({ site: pubUri, path: '/posts/hi' })
+      expect(
+        validateStandardSiteForUrl(
+          doc,
+          undefined,
+          'https://example.com/posts/hi',
+        ),
+      ).toBe(false)
+    })
+
+    it('rejects when assumedUrl is on a different host', () => {
+      const doc = makeDoc({ site: 'https://example.com', path: '/posts/hi' })
+      expect(
+        validateStandardSiteForUrl(doc, undefined, 'https://evil.com/posts/hi'),
+      ).toBe(false)
+    })
+
+    it('accepts when site has no path field and assumedUrl is the bare site', () => {
+      const doc = makeDoc({ site: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(doc, undefined, 'https://example.com'),
+      ).toBe(true)
+    })
+  })
+
+  describe('publication only', () => {
+    it('accepts when publication.url matches assumedUrl', () => {
+      const pub = makePub({ url: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(undefined, pub, 'https://example.com'),
+      ).toBe(true)
+    })
+
+    it('rejects when publication.url and assumedUrl differ', () => {
+      const pub = makePub({ url: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(undefined, pub, 'https://other.com'),
+      ).toBe(false)
+    })
+  })
+
+  describe('neither', () => {
+    it('returns true (caller handles the no-records short-circuit)', () => {
+      expect(
+        validateStandardSiteForUrl(undefined, undefined, 'https://example.com'),
+      ).toBe(true)
+    })
+  })
+
+  describe('assumedUrl variants', () => {
+    const doc = makeDoc({ site: pubUri, path: '/posts/hello' })
+    const pub = makePub({ url: 'https://example.com' })
+
+    for (const { note, assumedUrl, expected } of [
+      {
+        note: 'trailing slash on assumed URL',
+        assumedUrl: 'https://example.com/posts/hello/',
+        expected: true,
+      },
+      {
+        note: 'query string ignored',
+        assumedUrl: 'https://example.com/posts/hello?utm_source=twitter',
+        expected: true,
+      },
+      {
+        note: 'fragment ignored',
+        assumedUrl: 'https://example.com/posts/hello#section',
+        expected: true,
+      },
+      {
+        note: 'mixed-case host',
+        assumedUrl: 'https://Example.COM/posts/hello',
+        expected: true,
+      },
+      {
+        note: 'http vs https mismatch is rejected',
+        assumedUrl: 'http://example.com/posts/hello',
+        expected: false,
+      },
+      {
+        note: 'unparseable assumed URL is rejected',
+        assumedUrl: 'not a url',
+        expected: false,
+      },
+      {
+        note: 'non-http scheme is rejected',
+        assumedUrl: 'ftp://example.com/posts/hello',
+        expected: false,
+      },
+    ]) {
+      it(note, () => {
+        expect(validateStandardSiteForUrl(doc, pub, assumedUrl)).toBe(expected)
+      })
+    }
+  })
+
+  describe('path joining (root domain × subpath × slash placement)', () => {
+    // Real-world example that originally broke:
+    //   pub.url:  'https://atproto.com/blog'
+    //   doc.path: '/indexing-standard-site'
+    //   expected: 'https://atproto.com/blog/indexing-standard-site'
+    // `new URL('/indexing-standard-site', 'https://atproto.com/blog')` would
+    // resolve to 'https://atproto.com/indexing-standard-site' (the leading
+    // slash on path swallows the base's pathname under WHATWG semantics) —
+    // we want path-append, not relative resolution.
+    for (const { note, baseUrl, path, assumedUrl, expected } of [
+      // Root-domain base, all four slash combinations.
+      {
+        note: 'root base, base/, path/',
+        baseUrl: 'https://example.com/',
+        path: '/posts/hello',
+        assumedUrl: 'https://example.com/posts/hello',
+        expected: true,
+      },
+      {
+        note: 'root base, base/, path-no-slash',
+        baseUrl: 'https://example.com/',
+        path: 'posts/hello',
+        assumedUrl: 'https://example.com/posts/hello',
+        expected: true,
+      },
+      {
+        note: 'root base, base-no-slash, path/',
+        baseUrl: 'https://example.com',
+        path: '/posts/hello',
+        assumedUrl: 'https://example.com/posts/hello',
+        expected: true,
+      },
+      {
+        note: 'root base, base-no-slash, path-no-slash',
+        baseUrl: 'https://example.com',
+        path: 'posts/hello',
+        assumedUrl: 'https://example.com/posts/hello',
+        expected: true,
+      },
+      // Subpath base, all four slash combinations (the regression case).
+      {
+        note: 'subpath base, base/, path/',
+        baseUrl: 'https://atproto.com/blog/',
+        path: '/indexing-standard-site',
+        assumedUrl: 'https://atproto.com/blog/indexing-standard-site',
+        expected: true,
+      },
+      {
+        note: 'subpath base, base/, path-no-slash',
+        baseUrl: 'https://atproto.com/blog/',
+        path: 'indexing-standard-site',
+        assumedUrl: 'https://atproto.com/blog/indexing-standard-site',
+        expected: true,
+      },
+      {
+        note: 'subpath base, base-no-slash, path/ (regression)',
+        baseUrl: 'https://atproto.com/blog',
+        path: '/indexing-standard-site',
+        assumedUrl: 'https://atproto.com/blog/indexing-standard-site',
+        expected: true,
+      },
+      {
+        note: 'subpath base, base-no-slash, path-no-slash',
+        baseUrl: 'https://atproto.com/blog',
+        path: 'indexing-standard-site',
+        assumedUrl: 'https://atproto.com/blog/indexing-standard-site',
+        expected: true,
+      },
+      // Empty path: assumedUrl should equal the base.
+      {
+        note: 'empty path, root base, no trailing slash',
+        baseUrl: 'https://example.com',
+        path: undefined,
+        assumedUrl: 'https://example.com',
+        expected: true,
+      },
+      {
+        note: 'empty path, subpath base, with trailing slash',
+        baseUrl: 'https://atproto.com/blog/',
+        path: undefined,
+        assumedUrl: 'https://atproto.com/blog',
+        expected: true,
+      },
+      // Negative: subpath base with assumedUrl that lost the subpath
+      // (what `new URL`'s relative resolution would have produced).
+      {
+        note: 'rejects when assumed URL drops the subpath',
+        baseUrl: 'https://atproto.com/blog',
+        path: '/indexing-standard-site',
+        assumedUrl: 'https://atproto.com/indexing-standard-site',
+        expected: false,
+      },
+    ]) {
+      it(`doc + pub: ${note}`, () => {
+        const doc = makeDoc(
+          path === undefined ? { site: pubUri } : { site: pubUri, path },
+        )
+        const pub = makePub({ url: baseUrl })
+        expect(validateStandardSiteForUrl(doc, pub, assumedUrl)).toBe(expected)
+      })
+      it(`loose doc: ${note}`, () => {
+        const doc = makeDoc(
+          path === undefined ? { site: baseUrl } : { site: baseUrl, path },
+        )
+        expect(validateStandardSiteForUrl(doc, undefined, assumedUrl)).toBe(
+          expected,
+        )
+      })
+    }
+  })
+})