@atproto/bsky 0.0.232 → 0.0.234

package/src/util/standard-site.ts

@@ -0,0 +1,135 @@
+import { AtUriString } from '@atproto/syntax'
+
+export const SITE_STANDARD_NSID_PREFIX = 'site.standard.'
+
+/**
+ * Composes a stable map key from an `(uri, cid)` pair. A single hydration
+ * batch can pull more than one version of the same SS record URI (different
+ * posts pinning different cids), so the composite is needed for O(1)
+ * version-exact lookups.
+ */
+export const siteStandardRecordKey = (uri: string, cid: string) =>
+  `${uri}@${cid}`
+
+export const parseSiteStandardRecordKey = (
+  key: string,
+): { uri: AtUriString; cid: string } => {
+  const at = key.lastIndexOf('@')
+  return {
+    uri: key.slice(0, at) as AtUriString,
+    cid: key.slice(at + 1),
+  }
+}
+
+/**
+ * Parse `url` as HTTP(S) and reduce it to a canonical
+ * `protocol://host/path` string for equality comparison: lowercases host,
+ * strips a trailing slash from the path, and drops query/fragment. Returns
+ * `null` when the input isn't a valid HTTP(S) URL.
+ */
+const canonicalizeHttpUrl = (url: string): string | null => {
+  let parsed: URL
+  try {
+    parsed = new URL(url)
+  } catch {
+    return null
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null
+  const path = parsed.pathname === '/' ? '' : parsed.pathname.replace(/\/$/, '')
+  return `${parsed.protocol}//${parsed.host}${path}`
+}
+
+/**
+ * Append `path` to `base` with exactly one slash between, or return `base`
+ * unchanged when `path` is empty. Unlike `new URL(path, base)`, a leading
+ * slash on `path` does NOT swallow `base`'s pathname — so
+ * `joinPath('https://x.com/blog', '/foo')` is `https://x.com/blog/foo`,
+ * not `https://x.com/foo`.
+ */
+const joinPath = (base: string, path: string): string => {
+  if (!path) return base
+  const baseTrimmed = base.endsWith('/') ? base.slice(0, -1) : base
+  const pathTrimmed = path.startsWith('/') ? path.slice(1) : path
+  return `${baseTrimmed}/${pathTrimmed}`
+}
+
+/**
+ * Confirm that the supplied SS records actually back `assumedUrl`. The
+ * record-side URL is built by concatenating the publication URL (or the
+ * loose-doc site) with the document's `path` field, then both sides are
+ * canonicalized for equality: lowercase host, query/fragment dropped,
+ * trailing slash stripped.
+ *
+ * Path concatenation is `base + '/' + path` semantics — a leading `/` on
+ * `path` does NOT swallow the base's pathname (the way
+ * `new URL(path, base)` would). So
+ * `'https://atproto.com/blog' + '/indexing-standard-site'` resolves to
+ * `https://atproto.com/blog/indexing-standard-site` regardless of which
+ * side carries the slash.
+ *
+ * Structural validation of the doc/pub pair (matching `site` ↔ pub URI,
+ * no orphan docs that claim a missing publication) happens upstream in
+ * `getSiteStandardRecordsFromHydrationMapsByRefs` /
+ * `…ByDocumentUri` (see `hydration/external.ts`); by the time this
+ * function runs the pair is already known to be structurally consistent,
+ * so we only check whether the records back the URL.
+ *
+ * Cases:
+ * - Document + publication: `publication.url + document.path` must
+ *   canonicalize to `assumedUrl`.
+ * - Loose document (web-URL `site`): `document.site + document.path`
+ *   must canonicalize to `assumedUrl`. (Doc with at-uri `site` but no
+ *   publication can't reach this function — the lookups reject it.)
+ * - Publication only: `publication.url` must canonicalize to
+ *   `assumedUrl`.
+ * - Neither: vacuously valid; the caller short-circuits before we get
+ *   here.
+ */
+export const validateStandardSiteForUrl = (
+  document:
+    | {
+        ref: { uri: string }
+        info: { record: { site: string; path?: string } }
+      }
+    | undefined,
+  publication:
+    | { ref: { uri: string }; info: { record: { url: string } } }
+    | undefined,
+  assumedUrl: string,
+): boolean => {
+  const canonicalAssumed = canonicalizeHttpUrl(assumedUrl)
+  if (canonicalAssumed === null) return false
+
+  if (document && publication) {
+    const joined = canonicalizeHttpUrl(
+      joinPath(publication.info.record.url, document.info.record.path ?? ''),
+    )
+    return joined === canonicalAssumed
+  }
+  if (document) {
+    const joined = canonicalizeHttpUrl(
+      joinPath(document.info.record.site, document.info.record.path ?? ''),
+    )
+    return joined === canonicalAssumed
+  }
+  if (publication) {
+    return canonicalizeHttpUrl(publication.info.record.url) === canonicalAssumed
+  }
+  return true
+}
+
+const WORDS_PER_MINUTE = 200
+
+/**
+ * Estimate reading time in minutes from a plaintext document body. Returns
+ * `undefined` when the input has no countable words. Uses a coarse
+ * `WORDS_PER_MINUTE` heuristic; swap in a more accurate library here if
+ * needed (e.g. `reading-time`).
+ */
+export const estimateReadingTimeMinutes = (
+  text: string,
+): number | undefined => {
+  const words = text.trim().split(/\s+/).filter(Boolean).length
+  if (!words) return undefined
+  return Math.max(1, Math.ceil(words / WORDS_PER_MINUTE))
+}

package/src/views/index.ts

@@ -15,14 +15,25 @@ import {
   normalizeDatetimeAlways,
 } from '@atproto/syntax'
 import { Actor, ProfileViewerState } from '../hydration/actor.js'
+import {
+  AssociatedSiteStandardRecord,
+  SiteStandardDocument,
+  SiteStandardPublication,
+  getSiteStandardRecordsFromHydrationMapsByRefs,
+} from '../hydration/external.js'
 import { FeedItem, Like, Post, Repost } from '../hydration/feed.js'
 import { Follow, Verification } from '../hydration/graph.js'
 import { HydrationState } from '../hydration/hydrator.js'
 import { Label } from '../hydration/label.js'
 import { RecordInfo, parseString } from '../hydration/util.js'
 import { ImageUriBuilder } from '../image/uri.js'
-import { app } from '../lexicons/index.js'
+import { app, site } from '../lexicons/index.js'
+import { viewsLogger } from '../logger.js'
 import { Notification } from '../proto/bsky_pb.js'
+import {
+  estimateReadingTimeMinutes,
+  validateStandardSiteForUrl,
+} from '../util/standard-site.js'
 import {
   postUriToPostgateUri,
   postUriToThreadgateUri,
@@ -49,6 +60,9 @@ import {
   Embed,
   EmbedView,
   ExternalEmbed,
+  ExternalEmbedColorRgb,
+  ExternalEmbedSourceThemeView,
+  ExternalEmbedSourceView,
   ExternalEmbedView,
   FeedViewPost,
   FollowRecord,
@@ -87,6 +101,7 @@ import {
   RecordWithMediaView,
   ReplyRef,
   RepostRecord,
+  SiteStandardPublicationRecord,
   StarterPackView,
   StarterPackViewBasic,
   StatusView,
@@ -606,7 +621,7 @@ export class Views {
       status: record.status,
       embed:
         record.embed && isExternalEmbedType(record.embed)
-          ? this.externalEmbed(did, record.embed)
+          ? this.externalEmbed(did, record.embed, state)
           : undefined,
       labels,
       expiresAt,
@@ -2068,7 +2083,7 @@ export class Views {
     } else if (isVideoEmbedType(embed)) {
       return this.videoEmbed(creatorFromUri(postUri), embed)
     } else if (isExternalEmbedType(embed)) {
-      return this.externalEmbed(creatorFromUri(postUri), embed)
+      return this.externalEmbed(creatorFromUri(postUri), embed, state)
     } else if (isRecordEmbedType(embed)) {
       return this.recordEmbed(postUri, embed, state, depth)
     } else if (isRecordWithMediaType(embed)) {
@@ -2113,24 +2128,236 @@ export class Views {
   externalEmbed(
     did: DidString,
     embed: ExternalEmbed,
+    state: HydrationState,
   ): $Typed<ExternalEmbedView> {
-    const { uri, title, description, thumb } = embed.external
+    // Start from the post-author-supplied embed values, then spread the
+    // SS-derived view on top so any field the hydrated doc/publication
+    // supplies wins (title, description, thumb, source, etc). When no SS
+    // records were hydrated `ssView` is `undefined` and the spread is a
+    // no-op, leaving the base values in place. `associatedRefs` is set
+    // last because the post is authoritative about what was pinned.
+    const ssView = this.externalEmbedFromStandardSite({
+      associatedRefs: embed.external.associatedRefs,
+      state,
+      assumedUrl: embed.external.uri,
+    })
+    // Profiles of the owners of pinned `associatedRefs`. Hydrator covers
+    // these DIDs alongside post-author profiles, so misses here only
+    // happen when an actor is unavailable (suspended, deleted, etc.) —
+    // drop those rather than emit `undefined` slots.
+    const associatedProfiles = mapDefined(
+      embed.external.associatedRefs ?? [],
+      (ref) => this.profileBasic(uriToDid(ref.uri), state),
+    ) as ProfileViewBasic[]
     return app.bsky.embed.external.view.$build({
       external: {
-        uri,
-        title,
-        description,
-        thumb: thumb
+        uri: embed.external.uri,
+        title: embed.external.title,
+        description: embed.external.description,
+        thumb: embed.external.thumb
           ? this.imgUriBuilder.getPresetUri(
               'feed_thumbnail',
               did,
-              getBlobCidString(thumb),
+              getBlobCidString(embed.external.thumb),
             )
           : undefined,
+        ...ssView,
+        associatedRefs: embed.external.associatedRefs,
+        associatedProfiles: associatedProfiles.length
+          ? associatedProfiles
+          : undefined,
       },
     })
   }
 
+  /**
+   * Read-path entry point: caller has the post's `external.associatedRefs[]`
+   * and the global hydration state, and we resolve the matching SS records
+   * by ref. Used by `externalEmbed`.
+   *
+   * Returns a fully-populated `viewExternal['external']` shape (including
+   * the required `uri`/`title`/`description`), or `undefined` if the
+   * records didn't hydrate, didn't pass URL validation, or couldn't
+   * supply the required fields. Callers spread the result over a base
+   * view to layer SS-derived fields on top of post-author-supplied ones.
+   *
+   * `assumedUrl` is the canonical web URL the embed claims to represent
+   * (the post's `external.uri` on the read path, the request's `url` on
+   * compose). Validation logic uses it to confirm the SS records actually
+   * back that URL, and the same value is echoed back as `uri` on the
+   * returned overlay.
+   */
+  externalEmbedFromStandardSite({
+    associatedRefs,
+    state,
+    assumedUrl,
+  }: {
+    associatedRefs: ExternalEmbed['external']['associatedRefs']
+    state: HydrationState
+    assumedUrl: string
+  }): ExternalEmbedView['external'] | undefined {
+    const { document, publication } =
+      getSiteStandardRecordsFromHydrationMapsByRefs(
+        associatedRefs,
+        state.siteStandardDocuments,
+        state.siteStandardPublications,
+      )
+    return this.externalEmbedFromStandardSiteRecords({
+      document,
+      publication,
+      state,
+      assumedUrl,
+    })
+  }
+
+  /**
+   * Compose-path entry point: caller already knows which document and
+   * publication backed the request and has them in hand (e.g. from
+   * iterating the SS hydration maps directly). Skips the by-ref lookup
+   * that `externalEmbedFromStandardSite` does. Used by
+   * `getEmbedExternalView`.
+   *
+   * Returns a fully-populated `viewExternal['external']` shape (including
+   * the required `uri`/`title`/`description`), or `undefined` if any of
+   * the validation gates rejected the pair or the records couldn't
+   * supply the required fields.
+   *
+   * `state` is still needed for label hydration. `assumedUrl` is the
+   * canonical web URL the embed claims to represent (the post's
+   * `external.uri` on the read path, the request's `url` on compose);
+   * validation logic uses it to confirm the SS records actually back that
+   * URL, and the same value is echoed back as `uri` on the returned
+   * overlay.
+   */
+  externalEmbedFromStandardSiteRecords({
+    document,
+    publication,
+    state,
+    assumedUrl,
+  }: {
+    document: AssociatedSiteStandardRecord<SiteStandardDocument> | undefined
+    publication:
+      | AssociatedSiteStandardRecord<SiteStandardPublication>
+      | undefined
+    state: HydrationState
+    assumedUrl: string
+  }): ExternalEmbedView['external'] | undefined {
+    // Three layers of validation gate this overlay:
+    //
+    //  1. Hydrator nulls taken-down records and mirrors the null across
+    //     doc/pub pairs (see `actionSiteStandardTakedownLabels` in
+    //     `hydration/hydrator.ts`).
+    //  2. The lookups guarantee structural agreement between doc and pub
+    //     — matching `site`/`uri`, no orphan doc claiming a missing
+    //     publication (see `getSiteStandardRecordsFromHydrationMapsByRefs`
+    //     and `…ByDocumentUri` in `hydration/external.ts`).
+    //  3. `validateStandardSiteForUrl` below checks that the records back
+    //     the URL the embed claims (see `util/standard-site.ts`).
+    //
+    // If any layer rejected, callers see `undefined` and fall back to the
+    // bare embed rather than render partial / disagreeing enrichment.
+    if (!document && !publication) return undefined
+    if (!validateStandardSiteForUrl(document, publication, assumedUrl)) {
+      viewsLogger.warn(
+        {
+          documentUri: document?.ref.uri,
+          publicationUri: publication?.ref.uri,
+        },
+        'SS record(s) failed URL validation for external embed',
+      )
+      return undefined
+    }
+
+    // viewExternal requires `title` and `description`. If neither side of
+    // the pair supplies usable values for both, there's no enrichment to
+    // render — return undefined and let the caller fall back.
+    let title: string
+    let description: string
+    if (document?.info.record) {
+      // Treat the document as authoritative for both fields as a unit, so
+      // we never blend a doc's title with a publication's description.
+      title = document.info.record.title
+      description = document.info.record.description ?? ''
+    } else if (publication) {
+      title = publication.info.record.name
+      description = publication.info.record.description ?? ''
+    } else {
+      return undefined
+    }
+
+    // if we don't have a title at this point, something is wrong with the SS
+    // record (it's a required field) and therefore the enrichment isn't worth
+    // showing
+    if (!title) return undefined
+
+    const overlay: ExternalEmbedView['external'] = {
+      // @ts-ignore this is mis-typed
+      uri: assumedUrl,
+      title,
+      description,
+    }
+
+    const docCover = document?.info.record.coverImage
+    if (docCover) {
+      overlay.thumb = this.imgUriBuilder.getPresetUri(
+        'feed_thumbnail',
+        creatorFromUri(document.ref.uri),
+        getBlobCidString(docCover),
+      )
+    }
+
+    if (document?.info.record.publishedAt) {
+      overlay.createdAt = document.info.record.publishedAt
+    }
+    if (document?.info.record.updatedAt) {
+      overlay.updatedAt = document.info.record.updatedAt
+    }
+    const readingTime = document?.info.record.textContent
+      ? estimateReadingTimeMinutes(document.info.record.textContent)
+      : undefined
+    if (readingTime !== undefined) overlay.readingTime = readingTime
+
+    // Merge labels applied to either the document or the publication onto the
+    // same `viewExternal.labels` array — clients moderate the embed as single
+    // unit, so doc-scoped and publication-scoped labels end up in the same
+    // bucket.
+    const labels = [
+      ...(document ? state.labels?.getBySubject(document.ref.uri) ?? [] : []),
+      ...(publication
+        ? state.labels?.getBySubject(publication.ref.uri) ?? []
+        : []),
+    ]
+    if (labels.length) overlay.labels = labels
+
+    if (publication) {
+      overlay.source = this.externalEmbedSource(publication)
+    }
+
+    return overlay
+  }
+
+  externalEmbedSource(
+    publication: AssociatedSiteStandardRecord<SiteStandardPublication>,
+  ): $Typed<ExternalEmbedSourceView> {
+    const { record } = publication.info
+    const pubDid = creatorFromUri(publication.ref.uri)
+    return app.bsky.embed.external.viewExternalSource.$build({
+      uri: record.url,
+      icon: record.icon
+        ? this.imgUriBuilder.getPresetUri(
+            'avatar',
+            pubDid,
+            getBlobCidString(record.icon),
+          )
+        : undefined,
+      title: record.name,
+      description: record.description,
+      theme: record.basicTheme
+        ? externalEmbedSourceTheme(record.basicTheme)
+        : undefined,
+    })
+  }
+
   embedNotFound(uri: AtUriString): $Typed<RecordEmbedView> {
     return app.bsky.embed.record.view.$build({
       record: app.bsky.embed.record.viewNotFound.$build({
@@ -2289,7 +2516,7 @@ export class Views {
     } else if (isVideoEmbedType(embed.media)) {
       mediaEmbed = this.videoEmbed(creator, embed.media)
     } else if (isExternalEmbedType(embed.media)) {
-      mediaEmbed = this.externalEmbed(creator, embed.media)
+      mediaEmbed = this.externalEmbed(creator, embed.media, state)
     } else {
       return
     }
@@ -2458,3 +2685,31 @@ export class Views {
 const getRootUri = (uri: AtUriString, post: Post): AtUriString => {
   return post.record.reply?.root.uri ?? uri
 }
+
+const externalEmbedSourceTheme = (
+  theme: SiteStandardPublicationRecord['basicTheme'],
+): ExternalEmbedSourceThemeView | undefined => {
+  if (!theme) return undefined
+  const view: ExternalEmbedSourceThemeView = {}
+  const background = colorRGB(theme.background)
+  const foreground = colorRGB(theme.foreground)
+  const accent = colorRGB(theme.accent)
+  const accentForeground = colorRGB(theme.accentForeground)
+  if (background) view.backgroundRGB = background
+  if (foreground) view.foregroundRGB = foreground
+  if (accent) view.accentRGB = accent
+  if (accentForeground) view.accentForegroundRGB = accentForeground
+  // No fields set -> don't decorate the view at all.
+  return Object.keys(view).length === 0 ? undefined : view
+}
+
+const colorRGB = (
+  color: { $type?: unknown } | undefined,
+): ExternalEmbedColorRgb | undefined => {
+  if (!color || !site.standard.theme.color.rgb.isTypeOf(color)) return undefined
+  // `isTypeOf` narrows $type but not the shape; pull rgb fields off the
+  // unchecked input. Records that fail full validation are dropped above
+  // by the SS record parser, so by the time we get here the shape matches.
+  const { r, g, b } = color as unknown as { r: number; g: number; b: number }
+  return app.bsky.embed.external.colorRGB.$build({ r, g, b })
+}

package/src/views/types.ts

@@ -1,4 +1,4 @@
-import { app, chat, com } from '../lexicons/index.js'
+import { app, chat, com, site } from '../lexicons/index.js'
 
 // app.bsky.actor
 
@@ -36,6 +36,10 @@ export type VideoEmbedView = app.bsky.embed.video.View
 export const isExternalEmbedType = app.bsky.embed.external.$isTypeOf
 export type ExternalEmbed = app.bsky.embed.external.Main
 export type ExternalEmbedView = app.bsky.embed.external.View
+export type ExternalEmbedSourceView = app.bsky.embed.external.ViewExternalSource
+export type ExternalEmbedSourceThemeView =
+  app.bsky.embed.external.ViewExternalSourceTheme
+export type ExternalEmbedColorRgb = app.bsky.embed.external.ColorRgb
 
 export const isRecordEmbedType = app.bsky.embed.record.$isTypeOf
 export type RecordEmbed = app.bsky.embed.record.Main
@@ -173,3 +177,8 @@ export const validateStrongRef = com.atproto.repo.strongRef.$safeValidate
 // com.germnetwork.declaration
 
 export type GermDeclarationRecord = com.germnetwork.declaration.Main
+
+// site.standard
+
+export type SiteStandardDocumentRecord = site.standard.document.Main
+export type SiteStandardPublicationRecord = site.standard.publication.Main