@atproto/bsky 0.0.251 → 0.0.253

package/src/hydration/label.ts

@@ -1,207 +0,0 @@
-import { AtUriString, DidString, UriString, atUri } from '@atproto/lex'
-import { DataPlaneClient } from '../data-plane/client/index.js'
-import { app, com } from '../lexicons/index.js'
-import { ParsedLabelers } from '../util.js'
-import { Label, LabelerRecord } from '../views/types.js'
-import {
-  HydrationMap,
-  Merges,
-  RecordInfo,
-  parseJsonBytes,
-  parseRecord,
-  parseString,
-} from './util.js'
-
-export type { Label }
-
-export type SubjectLabels = {
-  isImpersonation: boolean
-  isTakendown: boolean
-  needsReview: boolean
-  labels: HydrationMap<`${string}::${string}`, Label> // src + val -> label
-}
-
-export class Labels
-  extends HydrationMap<UriString, SubjectLabels>
-  implements Merges
-{
-  static key(label: Label): `${string}::${string}` {
-    return `${label.src}::${label.val}`
-  }
-  merge(map: Labels): this {
-    for (const [key, theirs] of map) {
-      if (!theirs) continue
-      const mine = this.get(key)
-      if (mine) {
-        mine.isTakendown = mine.isTakendown || theirs.isTakendown
-        mine.labels = mine.labels.merge(theirs.labels)
-      } else {
-        this.set(key, theirs)
-      }
-    }
-    return this
-  }
-  getBySubject(sub: UriString): Label[] {
-    const it = this.get(sub)?.labels.values()
-    if (!it) return []
-    const labels: Label[] = []
-    for (const label of it) {
-      if (label) labels.push(label)
-    }
-    return labels
-  }
-}
-
-export type LabelerAgg = {
-  likes: number
-}
-
-export type LabelerAggs = HydrationMap<DidString, LabelerAgg>
-
-export type Labeler = RecordInfo<LabelerRecord>
-export type Labelers = HydrationMap<DidString, Labeler>
-
-export type LabelerViewerState = {
-  like?: AtUriString
-}
-
-export type LabelerViewerStates = HydrationMap<DidString, LabelerViewerState>
-
-export class LabelHydrator {
-  constructor(public dataplane: DataPlaneClient) {}
-
-  async getLabelsForSubjects(
-    subjects: UriString[],
-    labelers: ParsedLabelers,
-  ): Promise<Labels> {
-    const map = new Labels()
-
-    if (!subjects.length || !labelers.dids.length) return map
-
-    const res = await this.dataplane.getLabels({
-      subjects,
-      issuers: labelers.dids,
-    })
-
-    for (const cur of res.labels) {
-      const parsed = parseJsonBytes(com.atproto.label.defs.label, cur)
-      if (!parsed || parsed.neg) continue
-      const { sig: _, ...label } = parsed
-      let entry = map.get(label.uri)
-      if (!entry) {
-        entry = {
-          isImpersonation: false,
-          isTakendown: false,
-          needsReview: false,
-          labels: new HydrationMap(),
-        }
-        map.set(label.uri, entry)
-      }
-
-      const isActionableNeedsReview =
-        label.val === NEEDS_REVIEW_LABEL &&
-        !label.neg &&
-        labelers.redact.has(label.src)
-
-      // we action needs review labels on backend for now so don't send to client until client has proper logic for them
-      if (!isActionableNeedsReview) {
-        entry.labels.set(Labels.key(label), label)
-      }
-
-      if (
-        TAKEDOWN_LABELS.includes(label.val) &&
-        !label.neg &&
-        labelers.redact.has(label.src)
-      ) {
-        entry.isTakendown = true
-      }
-      if (isActionableNeedsReview) {
-        entry.needsReview = true
-      }
-      if (
-        label.val === IMPERSONATION_LABEL &&
-        !label.neg &&
-        labelers.redact.has(label.src)
-      ) {
-        entry.isImpersonation = true
-      }
-    }
-
-    return map
-  }
-
-  async getLabelers(
-    dids: DidString[],
-    includeTakedowns = false,
-  ): Promise<Labelers> {
-    const map: Labelers = new HydrationMap()
-    if (!dids.length) return map
-
-    const res = await this.dataplane.getLabelerRecords({
-      uris: dids.map(labelerDidToUri),
-    })
-    for (let i = 0; i < dids.length; i++) {
-      const did = dids[i]
-      const record = parseRecord(
-        app.bsky.labeler.service.main,
-        res.records[i],
-        includeTakedowns,
-      )
-      map.set(did, record ?? null)
-    }
-
-    return map
-  }
-
-  async getLabelerViewerStates(
-    dids: DidString[],
-    viewer: DidString,
-  ): Promise<LabelerViewerStates> {
-    const map: LabelerViewerStates = new HydrationMap()
-    if (!dids.length) return map
-
-    const likes = await this.dataplane.getLikesByActorAndSubjects({
-      actorDid: viewer,
-      refs: dids.map((did) => ({ uri: labelerDidToUri(did) })),
-    })
-
-    for (let i = 0; i < dids.length; i++) {
-      const did = dids[i]
-      map.set(did, {
-        like: parseString(likes.uris[i]),
-      })
-    }
-
-    return map
-  }
-
-  async getLabelerAggregates(
-    dids: DidString[],
-    viewer: DidString | null,
-  ): Promise<LabelerAggs> {
-    const map: LabelerAggs = new HydrationMap()
-    if (!dids.length) return map
-
-    const refs = dids.map((did) => ({ uri: labelerDidToUri(did) }))
-    const counts = await this.dataplane.getInteractionCounts({
-      refs,
-      skipCacheForDids: viewer ? [viewer] : undefined,
-    })
-    for (let i = 0; i < dids.length; i++) {
-      const did = dids[i]
-      map.set(did, {
-        likes: counts.likes[i] ?? 0,
-      })
-    }
-
-    return map
-  }
-}
-
-function labelerDidToUri<T extends DidString>(did: T) {
-  return atUri(did, app.bsky.labeler.service)
-}
-
-const IMPERSONATION_LABEL = 'impersonation'
-const TAKEDOWN_LABELS = ['!takedown', '!suspend']
-const NEEDS_REVIEW_LABEL = 'needs-review'

package/src/hydration/util.ts

@@ -1,199 +0,0 @@
-import { Timestamp } from '@bufbuild/protobuf'
-import {
-  AtUriString,
-  Cid,
-  InferInput,
-  InferOutput,
-  LexParseOptions,
-  LexValue,
-  RecordSchema,
-  Schema,
-  TypedLexMap,
-  ValidateOptions,
-  lexParseJsonBytes,
-  parseCidSafe,
-} from '@atproto/lex'
-import { AtUri } from '@atproto/syntax'
-import { Record as RecordEntry } from '../proto/bsky_pb.js'
-
-const PARSE_OPTIONS: LexParseOptions & ValidateOptions = {
-  strict: false,
-}
-
-export class HydrationMap<K, T> extends Map<K, T | null> implements Merges {
-  merge(map: HydrationMap<K, T>): this {
-    for (const [key, val] of map) {
-      this.set(key, val)
-    }
-    return this
-  }
-}
-
-export interface Merges {
-  merge<T extends this>(map: T): this
-}
-
-export type RecordInfo<T extends TypedLexMap> = {
-  record: T
-  cid: string
-  sortedAt: Date
-  indexedAt: Date
-  takedownRef: string | undefined
-}
-
-export const mergeMaps = <V extends HydrationMap<any, any>>(
-  mapA?: V,
-  mapB?: V,
-): V | undefined => {
-  if (!mapA) return mapB
-  if (!mapB) return mapA
-  return mapA.merge(mapB)
-}
-
-export const mergeNestedMaps = <K, V extends HydrationMap<any, any>>(
-  mapA?: HydrationMap<K, V>,
-  mapB?: HydrationMap<K, V>,
-): HydrationMap<K, V> | undefined => {
-  if (!mapA) return mapB
-  if (!mapB) return mapA
-
-  for (const [key, map] of mapB) {
-    const merged = mergeMaps(mapA.get(key) ?? undefined, map ?? undefined)
-    mapA.set(key, merged ?? null)
-  }
-
-  return mapA
-}
-
-export const mergeManyMaps = <K, T>(...maps: HydrationMap<K, T>[]) => {
-  return maps.reduce(mergeMaps, undefined as HydrationMap<K, T> | undefined)
-}
-
-export type ItemRef = { uri: AtUriString; cid?: string }
-
-export function parseRecord<TSchema extends RecordSchema>(
-  recordSchema: TSchema,
-  recordEntry: RecordEntry,
-  includeTakedowns: boolean,
-): RecordInfo<InferInput<TSchema>> | undefined {
-  if (!includeTakedowns && recordEntry.takenDown) {
-    return undefined
-  }
-
-  const cid = recordEntry.cid
-  if (!cid) {
-    return undefined
-  }
-
-  if (recordEntry.record.byteLength === 0) {
-    return undefined
-  }
-
-  const record = lexParseJsonBytes(recordEntry.record, PARSE_OPTIONS)
-  if (!record) {
-    return undefined
-  }
-
-  // @NOTE We cannot use parse mode here. We must return the original to ensure
-  // that the caller gets the same data as what is stored in the PDS (in case of
-  // records). This is important because the receiver of the data should be able
-  // to compute the right record CID.
-
-  if (!recordSchema.$matches(record, PARSE_OPTIONS)) {
-    return undefined
-  }
-
-  return {
-    record,
-    cid,
-    sortedAt: parseDate(recordEntry.sortedAt) ?? new Date(0),
-    indexedAt: parseDate(recordEntry.indexedAt) ?? new Date(0),
-    takedownRef: safeTakedownRef(recordEntry),
-  }
-}
-
-/**
- * Decodes binary data containing a JSON representation of a Lex value, and
- * validates it against the provided schema, in parse mode (i.e., allowing
- * coercion & defaults).
- *
- * Returns undefined if the input is empty (from dataplane's empty value
- * convention), or if the validation fails.
- */
-export const parseJsonBytes = <TSchema extends Schema<LexValue>>(
-  schema: TSchema,
-  bytes: Uint8Array | undefined,
-): InferOutput<TSchema> | undefined => {
-  if (!bytes || bytes.byteLength === 0) return undefined
-  const value = lexParseJsonBytes(bytes, PARSE_OPTIONS)
-  const result = schema.safeParse(value, PARSE_OPTIONS)
-  return result.success ? result.value : undefined
-}
-
-export const parseString = <T extends string | undefined>(
-  str: undefined | string,
-): T | undefined => {
-  return str ? (str as T) : undefined
-}
-
-export const parseCid = (cidStr: string | undefined): Cid | null => {
-  if (!cidStr) return null
-  return parseCidSafe(cidStr)
-}
-
-export const parseDate = (
-  timestamp: Timestamp | undefined,
-): Date | undefined => {
-  if (!timestamp) return undefined
-  const date = timestamp.toDate()
-  // Check for year 1 (0001-01-01 00:00:00 UTC) which is -62135596800000ms from epoch.
-  // The Go dataplane gives us those values as they come from the Go zero-value for dates.
-  if (date.getTime() === -62135596800000) return undefined
-  return date
-}
-
-export const urisByCollection = <T extends string>(
-  uris: Iterable<T>,
-): Map<string, T[]> => {
-  const result = new Map<string, T[]>()
-  for (const uri of uris) {
-    const collection = new AtUri(uri).collection
-    const items = result.get(collection) ?? []
-    items.push(uri)
-    result.set(collection, items)
-  }
-  return result
-}
-
-export const split = <T>(
-  items: T[],
-  predicate: (item: T) => boolean,
-): [T[], T[]] => {
-  const yes: T[] = []
-  const no: T[] = []
-  for (const item of items) {
-    if (predicate(item)) {
-      yes.push(item)
-    } else {
-      no.push(item)
-    }
-  }
-  return [yes, no]
-}
-
-export const safeTakedownRef = (obj?: {
-  takenDown: boolean
-  takedownRef: string
-}): string | undefined => {
-  if (!obj) return
-  if (obj.takedownRef) return obj.takedownRef
-  if (obj.takenDown) return 'BSKY-TAKEDOWN-UNKNOWN'
-}
-
-export const isActivitySubscriptionEnabled = ({
-  post,
-  reply,
-}: {
-  post: boolean
-  reply: boolean
-}): boolean => post || reply

package/src/image/index.ts

@@ -1,2 +0,0 @@
-export * from './sharp.js'
-export type { ImageInfo, Options } from './util.js'

package/src/image/invalidator.ts

@@ -1,33 +0,0 @@
-import { BlobCache } from './server.js'
-import { ImageUriBuilder } from './uri.js'
-
-// Invalidation is a general interface for propagating an image blob
-// takedown through any caches where a representation of it may be stored.
-// @NOTE this does not remove the blob from storage: just invalidates it from caches.
-// @NOTE keep in sync with same interface in aws/src/cloudfront.ts
-export interface ImageInvalidator {
-  invalidate(subject: string, paths: string[]): Promise<void>
-}
-
-export class ImageProcessingServerInvalidator implements ImageInvalidator {
-  constructor(private cache: BlobCache) {}
-  async invalidate(_subject: string, paths: string[]) {
-    const results = await Promise.allSettled(
-      paths.map(async (path) => {
-        const [, signature] = path.split('/')
-        if (!signature) throw new Error('Missing signature')
-        const options = ImageUriBuilder.getOptions(path)
-        const cacheKey = [
-          options.did,
-          options.cid.toString(),
-          options.preset,
-        ].join('::')
-        await this.cache.clear(cacheKey)
-      }),
-    )
-    const rejection = results.find(
-      (result): result is PromiseRejectedResult => result.status === 'rejected',
-    )
-    if (rejection) throw rejection.reason
-  }
-}

package/src/image/logger.ts

@@ -1,6 +0,0 @@
-import { subsystemLogger } from '@atproto/common'
-
-export const logger: ReturnType<typeof subsystemLogger> =
-  subsystemLogger('bsky:image')
-
-export default logger

package/src/image/server.ts

@@ -1,228 +0,0 @@
-import fsSync from 'node:fs'
-import fs from 'node:fs/promises'
-import os from 'node:os'
-import path from 'node:path'
-import { Duplex, Readable } from 'node:stream'
-import { pipeline } from 'node:stream/promises'
-import createError, { isHttpError } from 'http-errors'
-import {
-  VerifyCidError,
-  VerifyCidTransform,
-  cloneStream,
-  createDecoders,
-  isErrnoException,
-} from '@atproto/common'
-import { BlobNotFoundError } from '@atproto/repo'
-import { StreamBlobOptions, streamBlob } from '../api/blob-resolver.js'
-import { AppContext } from '../context.js'
-import { Middleware, responseSignal } from '../util/http.js'
-import log from './logger.js'
-import { createImageProcessor, createImageUpscaler } from './sharp.js'
-import { BadPathError, ImageUriBuilder } from './uri.js'
-import { Options, SharpInfo, formatsToMimes } from './util.js'
-
-export function createMiddleware(
-  ctx: AppContext,
-  { prefix = '/' }: { prefix?: string } = {},
-): Middleware {
-  if (!prefix.startsWith('/') || !prefix.endsWith('/')) {
-    throw new TypeError('Prefix must start and end with a slash')
-  }
-
-  // If there is a CDN, we don't need to serve images
-  if (ctx.cfg.cdnUrl) {
-    return (req, res, next) => next()
-  }
-
-  const cache = new BlobDiskCache(ctx.cfg.blobCacheLocation)
-
-  return async (req, res, next) => {
-    if (res.destroyed) return
-    if (req.method !== 'GET' && req.method !== 'HEAD') return next()
-    if (!req.url?.startsWith(prefix)) return next()
-    const { 0: path, 1: _search } = req.url.slice(prefix.length - 1).split('?')
-    if (!path.startsWith('/') || path === '/') return next()
-
-    try {
-      const options = ImageUriBuilder.getOptions(path)
-
-      const cacheKey = [options.did, options.cid, options.preset].join('::')
-
-      // Cached flow
-
-      try {
-        const cachedImage = await cache.get(cacheKey)
-        res.statusCode = 200
-        res.setHeader('x-cache', 'hit')
-        res.setHeader('content-type', getMime(options.format))
-        res.setHeader('cache-control', `public, max-age=31536000`) // 1 year
-        res.setHeader('content-length', cachedImage.size)
-        await pipeline(cachedImage, res)
-        return
-      } catch (err) {
-        if (!(err instanceof BlobNotFoundError)) {
-          log.error({ cacheKey, err }, 'failed to serve cached image')
-        }
-
-        if (res.headersSent || res.destroyed) {
-          res.destroy()
-          return // nothing we can do...
-        } else {
-          // Ignore and move on to non-cached flow.
-          res.removeHeader('x-cache')
-          res.removeHeader('content-type')
-          res.removeHeader('cache-control')
-          res.removeHeader('content-length')
-        }
-      }
-
-      // Non-cached flow
-
-      const streamOptions: StreamBlobOptions = {
-        did: options.did,
-        cid: options.cid,
-        signal: responseSignal(res),
-      }
-
-      await streamBlob(ctx, streamOptions, (upstream, { did, cid, url }) => {
-        // Definitely not an image ? Let's fail right away.
-        if (isImageMime(upstream.headers['content-type']) === false) {
-          throw createError(400, 'Not an image')
-        }
-
-        // Let's transform (decompress, verify CID, upscale), process and respond
-
-        const transforms: Duplex[] = [
-          ...createDecoders(upstream.headers['content-encoding']),
-          new VerifyCidTransform(cid),
-          createImageUpscaler(options),
-        ]
-        const processor = createImageProcessor(options)
-
-        // Cache in the background
-        cache
-          .put(cacheKey, cloneStream(processor))
-          .catch((err) => log.error({ err }, 'failed to cache image'))
-
-        res.statusCode = 200
-        res.setHeader('cache-control', `public, max-age=31536000`) // 1 year
-        res.setHeader('x-cache', 'miss')
-        processor.once('info', ({ size, format }: SharpInfo) => {
-          const type = formatsToMimes.get(format) || 'application/octet-stream'
-
-          // @NOTE sharp does emit this in time to be set as a header
-          res.setHeader('content-length', size)
-          res.setHeader('content-type', type)
-        })
-
-        const streams = [...transforms, processor, res]
-        void pipeline(streams).catch((err: unknown) => {
-          log.warn(
-            { err, did, cid: cid.toString(), pds: url.origin },
-            'blob resolution failed during transmission',
-          )
-        })
-
-        return streams[0]!
-      })
-    } catch (err) {
-      if (res.headersSent || res.destroyed) {
-        res.destroy()
-      } else {
-        res.removeHeader('content-type')
-        res.removeHeader('content-length')
-        res.removeHeader('cache-control')
-        res.removeHeader('x-cache')
-
-        if (err instanceof BadPathError) {
-          next(createError(400, err))
-        } else if (err instanceof VerifyCidError) {
-          next(createError(404, 'Blob not found', err))
-        } else if (isHttpError(err)) {
-          next(err)
-        } else {
-          next(createError(502, 'Upstream Error', { cause: err }))
-        }
-      }
-    }
-  }
-}
-
-function isImageMime(
-  contentType: string | string[] | undefined,
-): undefined | boolean {
-  if (contentType == null || contentType === 'application/octet-stream') {
-    return undefined // maybe
-  }
-  if (Array.isArray(contentType)) {
-    if (contentType.length === 0) return undefined // should never happen
-    if (contentType.length === 1) return isImageMime(contentType[0])
-    return contentType.every(isImageMime) // Should we throw a 502 here?
-  }
-  return contentType.startsWith('image/')
-}
-
-function getMime(format: Options['format']) {
-  const mime = formatsToMimes.get(format)
-  if (!mime) throw new Error('Unknown format')
-  return mime
-}
-
-export interface BlobCache {
-  get(fileId: string): Promise<Readable & { size: number }>
-  put(fileId: string, stream: Readable): Promise<void>
-  clear(fileId: string): Promise<void>
-  clearAll(): Promise<void>
-}
-
-export class BlobDiskCache implements BlobCache {
-  tempDir: string
-  constructor(basePath?: string) {
-    this.tempDir = basePath || path.join(os.tmpdir(), 'bsky--processed-images')
-    if (!path.isAbsolute(this.tempDir)) {
-      throw new Error('Must provide an absolute path')
-    }
-    try {
-      fsSync.mkdirSync(this.tempDir, { recursive: true })
-    } catch (err) {
-      // All good if cache dir already exists
-      if (isErrnoException(err) && err.code === 'EEXIST') return
-    }
-  }
-
-  async get(fileId: string) {
-    try {
-      const handle = await fs.open(path.join(this.tempDir, fileId), 'r')
-      const { size } = await handle.stat()
-      if (size === 0) {
-        throw new BlobNotFoundError()
-      }
-      return Object.assign(handle.createReadStream(), { size })
-    } catch (err) {
-      if (isErrnoException(err) && err.code === 'ENOENT') {
-        throw new BlobNotFoundError()
-      }
-      throw err
-    }
-  }
-
-  async put(fileId: string, stream: Readable) {
-    const filename = path.join(this.tempDir, fileId)
-    try {
-      await fs.writeFile(filename, stream, { flag: 'wx' })
-    } catch (err) {
-      // Do not overwrite existing file, just ignore the error
-      if (isErrnoException(err) && err.code === 'EEXIST') return
-      throw err
-    }
-  }
-
-  async clear(fileId: string) {
-    const filename = path.join(this.tempDir, fileId)
-    await fs.rm(filename, { force: true })
-  }
-
-  async clearAll() {
-    await fs.rm(this.tempDir, { recursive: true, force: true })
-  }
-}