@atproto/bsky 0.0.231 → 0.0.233

package/src/hydration/hydrator.ts

@@ -1,5 +1,5 @@
 import assert from 'node:assert'
-import { mapDefined } from '@atproto/common'
+import { dedupeStrs, mapDefined } from '@atproto/common'
 import { AtUri, AtUriString, DidString, UriString } from '@atproto/syntax'
 import { DataPlaneClient } from '../data-plane/client/index.js'
 import {
@@ -14,10 +14,15 @@ import {
   Notification,
   RecordRef,
 } from '../proto/bsky_pb.js'
+import {
+  SITE_STANDARD_NSID_PREFIX,
+  parseSiteStandardRecordKey,
+} from '../util/standard-site.js'
 import { uriToDid, uriToDid as didFromUri } from '../util/uris.js'
 import { ParsedLabelers } from '../util.js'
 import {
   ProfileRecord,
+  isExternalEmbedType,
   isListRuleType,
   isRecordEmbedType,
   isRecordWithMediaType,
@@ -31,6 +36,11 @@ import {
   ProfileViewerState,
   ProfileViewerStates,
 } from './actor.js'
+import {
+  ExternalHydrator,
+  SiteStandardDocuments,
+  SiteStandardPublications,
+} from './external.js'
 import {
   FeedGenAggs,
   FeedGenViewerStates,
@@ -151,6 +161,8 @@ export type HydrationState = {
   bidirectionalBlocks?: BidirectionalBlocks
   verifications?: Verifications
   bookmarks?: Bookmarks
+  siteStandardDocuments?: SiteStandardDocuments
+  siteStandardPublications?: SiteStandardPublications
 }
 
 export type PostBlock = { embed: boolean; parent: boolean; root: boolean }
@@ -193,6 +205,7 @@ export type HydratorConfig = {
 
 export class Hydrator {
   actor: ActorHydrator
+  external: ExternalHydrator
   feed: FeedHydrator
   graph: GraphHydrator
   label: LabelHydrator
@@ -206,6 +219,7 @@ export class Hydrator {
   ) {
     this.config = config
     this.actor = new ActorHydrator(dataplane)
+    this.external = new ExternalHydrator(dataplane)
     this.feed = new FeedHydrator(dataplane)
     this.graph = new GraphHydrator(dataplane)
     this.label = new LabelHydrator(dataplane)
@@ -590,6 +604,15 @@ export class Hydrator {
       }
     }
 
+    const siteStandardRefs = siteStandardRefsFromPosts(
+      postsLayer0,
+      postsLayer1,
+      postsLayer2,
+    )
+    const siteStandardLabelSubjects = dedupeStrs(
+      siteStandardRefs.map((ref) => ref.uri),
+    )
+
     const [
       postAggs,
       postViewers,
@@ -601,12 +624,19 @@ export class Hydrator {
       labelerState,
       starterPackState,
       postgates,
+      {
+        documents: siteStandardDocuments,
+        publications: siteStandardPublications,
+      },
     ] = await Promise.all([
       this.feed.getPostAggregates(allRefs, ctx.viewer),
       ctx.viewer
         ? this.feed.getPostViewerStates(threadRefs, ctx.viewer)
         : undefined,
-      this.label.getLabelsForSubjects(allPostUris, ctx.labelers),
+      this.label.getLabelsForSubjects(
+        [...allPostUris, ...siteStandardLabelSubjects],
+        ctx.labelers,
+      ),
       this.hydratePostBlocks(posts, ctx),
       this.hydrateProfiles(allPostUris.map(didFromUri), ctx),
       this.hydrateLists([...nestedListUris, ...threadgateListUris], ctx),
@@ -614,9 +644,18 @@ export class Hydrator {
       this.hydrateLabelers(nestedLabelerDids, ctx),
       this.hydrateStarterPacksBasic(nestedStarterPackUris, ctx),
       this.feed.getPostgatesForPosts([...postUrisWithPostgates.values()]),
+      this.external.getSiteStandardRecordsByRef(
+        siteStandardRefs,
+        ctx.includeTakedowns,
+      ),
     ])
     if (!ctx.includeTakedowns) {
       actionTakedownLabels(allPostUris, posts, labels)
+      actionSiteStandardTakedownLabels(
+        siteStandardDocuments,
+        siteStandardPublications,
+        labels,
+      )
     }
     // combine all hydration state
     return mergeManyStates(
@@ -633,6 +672,8 @@ export class Hydrator {
         labels,
         threadgates,
         postgates,
+        siteStandardDocuments,
+        siteStandardPublications,
         ctx,
       },
     )
@@ -753,6 +794,38 @@ export class Hydrator {
     })
   }
 
+  /**
+   * Hydrate the state needed to build an `app.bsky.embed.external#view` for
+   * a set of `site.standard.*` AT-URIs at compose-time. Filters input URIs to
+   * SS collections, fetches latest indexed versions plus labels, and gates
+   * taken-down records.
+   */
+  async hydrateEmbedExternalViewFromUris(
+    uris: AtUriString[],
+    ctx: HydrateCtx,
+  ): Promise<HydrationState> {
+    const ssUris = dedupeStrs(
+      uris.filter((u) =>
+        new AtUri(u).collection.startsWith(SITE_STANDARD_NSID_PREFIX),
+      ),
+    ) as AtUriString[]
+    if (!ssUris.length) return { ctx }
+
+    const [{ documents, publications }, labels] = await Promise.all([
+      this.external.getSiteStandardRecordsByURI(ssUris, ctx.includeTakedowns),
+      this.label.getLabelsForSubjects(ssUris, ctx.labelers),
+    ])
+    if (!ctx.includeTakedowns) {
+      actionSiteStandardTakedownLabels(documents, publications, labels)
+    }
+    return {
+      ctx,
+      labels,
+      siteStandardDocuments: documents,
+      siteStandardPublications: publications,
+    }
+  }
+
   // app.bsky.feed.defs#threadViewPost
   // - post
   //   - profile
@@ -1480,6 +1553,55 @@ const nestedRecordUris = (post: Post['record']): AtUriString[] => {
   return uris
 }
 
+/**
+ * Returns every strongRef the post embedded as `external.associatedRefs`,
+ * regardless of collection. Callers are responsible for filtering to NSIDs
+ * they care about.
+ */
+const externalAssociatedRefs = (
+  post: Post['record'],
+): { uri: AtUriString; cid: string }[] => {
+  const embed = post?.embed
+  if (!embed) return []
+  if (isExternalEmbedType(embed)) {
+    return embed.external.associatedRefs ?? []
+  }
+  if (isRecordWithMediaType(embed) && isExternalEmbedType(embed.media)) {
+    return embed.media.external.associatedRefs ?? []
+  }
+  return []
+}
+
+/**
+ * Collects standard.site refs across all post layers, deduped by `uri+cid` so
+ * the dataplane batch is minimal even if multiple posts in the layer reference
+ * the same exact version of an SS record. The same `${uri}@${cid}` keys are
+ * what `getSiteStandardRecordsByRef` uses on the resulting hydration maps,
+ * which lets `lookupAssociatedSiteStandardRecords` do an O(1) version-exact
+ * lookup at view time.
+ */
+const siteStandardRefsFromPosts = (...postLayers: Posts[]): ItemRef[] => {
+  const seen = new Set<string>()
+  const out: ItemRef[] = []
+  for (const layer of postLayers) {
+    for (const item of layer.values()) {
+      if (!item) continue
+      for (const ref of externalAssociatedRefs(item.record)) {
+        if (
+          !new AtUri(ref.uri).collection.startsWith(SITE_STANDARD_NSID_PREFIX)
+        ) {
+          continue
+        }
+        const key = `${ref.uri}@${ref.cid}`
+        if (seen.has(key)) continue
+        seen.add(key)
+        out.push({ uri: ref.uri, cid: ref.cid })
+      }
+    }
+  }
+  return out
+}
+
 const getListUrisFromThreadgates = (gates: Threadgates): AtUriString[] => {
   const uris: AtUriString[] = []
   for (const gate of gates.values()) {
@@ -1560,6 +1682,14 @@ export const mergeStates = (
     ),
     verifications: mergeMaps(stateA.verifications, stateB.verifications),
     bookmarks: mergeNestedMaps(stateA.bookmarks, stateB.bookmarks),
+    siteStandardDocuments: mergeMaps(
+      stateA.siteStandardDocuments,
+      stateB.siteStandardDocuments,
+    ),
+    siteStandardPublications: mergeMaps(
+      stateA.siteStandardPublications,
+      stateB.siteStandardPublications,
+    ),
   }
 }
 
@@ -1579,6 +1709,57 @@ const actionTakedownLabels = (
   }
 }
 
+/**
+ * Apply takedown labels to the site.standard hydration maps. Per-record
+ * takedowns null any entry whose subject URI is taken down; pair takedowns
+ * propagate that null across doc/publication pairs so we never render half
+ * of a moderated embed. Pairs are discovered via `doc.record.site`; orphan
+ * docs and orphan publications are subject only to the per-record sweep.
+ */
+const actionSiteStandardTakedownLabels = (
+  documents: SiteStandardDocuments,
+  publications: SiteStandardPublications,
+  labels: Labels,
+) => {
+  // Pairings have to be captured before nulling — the doc record carries
+  // the publication URI in its `site` field, and we lose it once we null.
+  const pairings: { docKey: string; pubKey: string }[] = []
+  if (documents.size > 0 && publications.size > 0) {
+    const pubKeysByUri = new Map<string, string[]>()
+    for (const key of publications.keys()) {
+      const { uri } = parseSiteStandardRecordKey(key)
+      const list = pubKeysByUri.get(uri)
+      if (list) list.push(key)
+      else pubKeysByUri.set(uri, [key])
+    }
+    for (const [docKey, doc] of documents) {
+      const site = doc?.record.site
+      if (!site || !site.startsWith('at://')) continue
+      for (const pubKey of pubKeysByUri.get(site) ?? []) {
+        pairings.push({ docKey, pubKey })
+      }
+    }
+  }
+
+  // Per-record takedowns: null any entry whose subject URI is taken down.
+  for (const key of documents.keys()) {
+    const { uri } = parseSiteStandardRecordKey(key)
+    if (labels.get(uri)?.isTakendown) documents.set(key, null)
+  }
+  for (const key of publications.keys()) {
+    const { uri } = parseSiteStandardRecordKey(key)
+    if (labels.get(uri)?.isTakendown) publications.set(key, null)
+  }
+
+  // Pair takedowns: if exactly one side of a pair was nulled above, mirror.
+  for (const { docKey, pubKey } of pairings) {
+    const doc = documents.get(docKey)
+    const pub = publications.get(pubKey)
+    if (doc === null && pub !== null) publications.set(pubKey, null)
+    if (pub === null && doc !== null) documents.set(docKey, null)
+  }
+}
+
 const uriToRef = (uri: AtUriString): ItemRef => {
   return { uri }
 }

package/src/util/standard-site.test.ts

@@ -0,0 +1,186 @@
+import { describe, expect, it } from 'vitest'
+import { validateStandardSiteForUrl } from './standard-site.js'
+
+const docDid = 'did:plc:doc'
+const pubDid = 'did:plc:pub'
+const docUri = `at://${docDid}/site.standard.document/abc`
+const pubUri = `at://${pubDid}/site.standard.publication/self`
+
+const makeDoc = (record: { site: string; path?: string }) => ({
+  ref: { uri: docUri },
+  info: { record },
+})
+
+const makePub = (record: { url: string }) => ({
+  ref: { uri: pubUri },
+  info: { record },
+})
+
+describe(validateStandardSiteForUrl, () => {
+  describe('document + publication', () => {
+    it('accepts when site resolves to publication and url+path matches', () => {
+      const doc = makeDoc({ site: pubUri, path: '/posts/hello' })
+      const pub = makePub({ url: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(doc, pub, 'https://example.com/posts/hello'),
+      ).toBe(true)
+    })
+
+    it('rejects when joined url+path does not match assumedUrl', () => {
+      const doc = makeDoc({ site: pubUri, path: '/posts/hello' })
+      const pub = makePub({ url: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(
+          doc,
+          pub,
+          'https://example.com/posts/different',
+        ),
+      ).toBe(false)
+    })
+
+    it('accepts when document has no path and assumedUrl is the bare publication url', () => {
+      const doc = makeDoc({ site: pubUri })
+      const pub = makePub({ url: 'https://example.com' })
+      expect(validateStandardSiteForUrl(doc, pub, 'https://example.com')).toBe(
+        true,
+      )
+    })
+  })
+
+  describe('document only (loose doc)', () => {
+    it('accepts when site is HTTP and matches assumedUrl', () => {
+      const doc = makeDoc({ site: 'https://example.com', path: '/posts/hi' })
+      expect(
+        validateStandardSiteForUrl(
+          doc,
+          undefined,
+          'https://example.com/posts/hi',
+        ),
+      ).toBe(true)
+    })
+
+    // The lookups reject this pairing upstream, but validate as a fallback:
+    // an at-uri base canonicalizes to null and fails the URL match.
+    it('rejects when site is an at-uri (non-HTTP base canonicalizes to null)', () => {
+      const doc = makeDoc({ site: pubUri, path: '/posts/hi' })
+      expect(
+        validateStandardSiteForUrl(
+          doc,
+          undefined,
+          'https://example.com/posts/hi',
+        ),
+      ).toBe(false)
+    })
+
+    it('rejects when assumedUrl is on a different host', () => {
+      const doc = makeDoc({ site: 'https://example.com', path: '/posts/hi' })
+      expect(
+        validateStandardSiteForUrl(doc, undefined, 'https://evil.com/posts/hi'),
+      ).toBe(false)
+    })
+
+    it('accepts when site has no path field and assumedUrl is the bare site', () => {
+      const doc = makeDoc({ site: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(doc, undefined, 'https://example.com'),
+      ).toBe(true)
+    })
+  })
+
+  describe('publication only', () => {
+    it('accepts when publication.url matches assumedUrl', () => {
+      const pub = makePub({ url: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(undefined, pub, 'https://example.com'),
+      ).toBe(true)
+    })
+
+    it('rejects when publication.url and assumedUrl differ', () => {
+      const pub = makePub({ url: 'https://example.com' })
+      expect(
+        validateStandardSiteForUrl(undefined, pub, 'https://other.com'),
+      ).toBe(false)
+    })
+  })
+
+  describe('neither', () => {
+    it('returns true (caller handles the no-records short-circuit)', () => {
+      expect(
+        validateStandardSiteForUrl(undefined, undefined, 'https://example.com'),
+      ).toBe(true)
+    })
+  })
+
+  describe('assumedUrl variants', () => {
+    const doc = makeDoc({ site: pubUri, path: '/posts/hello' })
+    const pub = makePub({ url: 'https://example.com' })
+
+    for (const { note, assumedUrl, expected } of [
+      {
+        note: 'trailing slash on assumed URL',
+        assumedUrl: 'https://example.com/posts/hello/',
+        expected: true,
+      },
+      {
+        note: 'query string ignored',
+        assumedUrl: 'https://example.com/posts/hello?utm_source=twitter',
+        expected: true,
+      },
+      {
+        note: 'fragment ignored',
+        assumedUrl: 'https://example.com/posts/hello#section',
+        expected: true,
+      },
+      {
+        note: 'mixed-case host',
+        assumedUrl: 'https://Example.COM/posts/hello',
+        expected: true,
+      },
+      {
+        note: 'http vs https mismatch is rejected',
+        assumedUrl: 'http://example.com/posts/hello',
+        expected: false,
+      },
+      {
+        note: 'unparseable assumed URL is rejected',
+        assumedUrl: 'not a url',
+        expected: false,
+      },
+      {
+        note: 'non-http scheme is rejected',
+        assumedUrl: 'ftp://example.com/posts/hello',
+        expected: false,
+      },
+    ]) {
+      it(note, () => {
+        expect(validateStandardSiteForUrl(doc, pub, assumedUrl)).toBe(expected)
+      })
+    }
+  })
+
+  describe('publication.url variants', () => {
+    it('accepts when publication.url has trailing slash', () => {
+      const doc = makeDoc({ site: pubUri, path: '/posts/hello' })
+      const pub = makePub({ url: 'https://example.com/' })
+      expect(
+        validateStandardSiteForUrl(doc, pub, 'https://example.com/posts/hello'),
+      ).toBe(true)
+    })
+
+    it('accepts when document.path lacks a leading slash', () => {
+      const doc = makeDoc({ site: pubUri, path: 'posts/hello' })
+      const pub = makePub({ url: 'https://example.com/' })
+      expect(
+        validateStandardSiteForUrl(doc, pub, 'https://example.com/posts/hello'),
+      ).toBe(true)
+    })
+
+    it('handles publication.url with sub-path and document.path joining onto it', () => {
+      const doc = makeDoc({ site: pubUri, path: 'hello' })
+      const pub = makePub({ url: 'https://example.com/blog/' })
+      expect(
+        validateStandardSiteForUrl(doc, pub, 'https://example.com/blog/hello'),
+      ).toBe(true)
+    })
+  })
+})

package/src/util/standard-site.ts

@@ -0,0 +1,116 @@
+import { AtUriString } from '@atproto/syntax'
+
+export const SITE_STANDARD_NSID_PREFIX = 'site.standard.'
+
+/**
+ * Composes a stable map key from an `(uri, cid)` pair. A single hydration
+ * batch can pull more than one version of the same SS record URI (different
+ * posts pinning different cids), so the composite is needed for O(1)
+ * version-exact lookups.
+ */
+export const siteStandardRecordKey = (uri: string, cid: string) =>
+  `${uri}@${cid}`
+
+export const parseSiteStandardRecordKey = (
+  key: string,
+): { uri: AtUriString; cid: string } => {
+  const at = key.lastIndexOf('@')
+  return {
+    uri: key.slice(0, at) as AtUriString,
+    cid: key.slice(at + 1),
+  }
+}
+
+/**
+ * Parse `url` as HTTP(S) and reduce it to a canonical
+ * `protocol://host/path` string for equality comparison: lowercases host,
+ * strips a trailing slash from the path, and drops query/fragment. Returns
+ * `null` when the input isn't a valid HTTP(S) URL.
+ */
+const canonicalizeHttpUrl = (url: string, base?: string): string | null => {
+  let parsed: URL
+  try {
+    parsed = new URL(url, base)
+  } catch {
+    return null
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null
+  const path = parsed.pathname === '/' ? '' : parsed.pathname.replace(/\/$/, '')
+  return `${parsed.protocol}//${parsed.host}${path}`
+}
+
+/**
+ * Confirm that the supplied SS records actually back `assumedUrl`. URL
+ * comparison parses both sides via the WHATWG URL constructor (resolving
+ * the document's `path` against the publication or site as a base) and
+ * compares the canonical `protocol://host/path` forms — host case is
+ * ignored, query/fragment are dropped, trailing slashes stripped.
+ *
+ * Structural validation of the doc/pub pair (matching `site` ↔ pub URI,
+ * no orphan docs that claim a missing publication) happens upstream in
+ * `getSiteStandardRecordsFromHydrationMapsByRefs` /
+ * `…ByDocumentUri` (see `hydration/external.ts`); by the time this
+ * function runs the pair is already known to be structurally consistent,
+ * so we only check whether the records back the URL.
+ *
+ * Cases:
+ * - Document + publication: `publication.url + document.path` must
+ *   canonicalize to `assumedUrl`.
+ * - Loose document (web-URL `site`): `document.site + document.path`
+ *   must canonicalize to `assumedUrl`. (Doc with at-uri `site` but no
+ *   publication can't reach this function — the lookups reject it.)
+ * - Publication only: `publication.url` must canonicalize to
+ *   `assumedUrl`.
+ * - Neither: vacuously valid; the caller short-circuits before we get
+ *   here.
+ */
+export const validateStandardSiteForUrl = (
+  document:
+    | {
+        ref: { uri: string }
+        info: { record: { site: string; path?: string } }
+      }
+    | undefined,
+  publication:
+    | { ref: { uri: string }; info: { record: { url: string } } }
+    | undefined,
+  assumedUrl: string,
+): boolean => {
+  const canonicalAssumed = canonicalizeHttpUrl(assumedUrl)
+  if (canonicalAssumed === null) return false
+
+  if (document && publication) {
+    const joined = canonicalizeHttpUrl(
+      document.info.record.path ?? '',
+      publication.info.record.url,
+    )
+    return joined === canonicalAssumed
+  }
+  if (document) {
+    const joined = canonicalizeHttpUrl(
+      document.info.record.path ?? '',
+      document.info.record.site,
+    )
+    return joined === canonicalAssumed
+  }
+  if (publication) {
+    return canonicalizeHttpUrl(publication.info.record.url) === canonicalAssumed
+  }
+  return true
+}
+
+const WORDS_PER_MINUTE = 200
+
+/**
+ * Estimate reading time in minutes from a plaintext document body. Returns
+ * `undefined` when the input has no countable words. Uses a coarse
+ * `WORDS_PER_MINUTE` heuristic; swap in a more accurate library here if
+ * needed (e.g. `reading-time`).
+ */
+export const estimateReadingTimeMinutes = (
+  text: string,
+): number | undefined => {
+  const words = text.trim().split(/\s+/).filter(Boolean).length
+  if (!words) return undefined
+  return Math.max(1, Math.ceil(words / WORDS_PER_MINUTE))
+}