@atproto/pds 0.5.11 → 0.5.14

package/src/rate-limits.ts

@@ -1,59 +0,0 @@
-import type { Redis } from 'ioredis'
-import { DAY, HOUR, MINUTE } from '@atproto/common'
-import { MemoryRateLimiter, RedisRateLimiter } from '@atproto/xrpc-server'
-import type { Options } from '@atproto/xrpc-server'
-import type { RateLimitsConfig } from './config/index.js'
-
-type RateLimitDescriptions = NonNullable<Options['rateLimits']>
-
-const SYNC_GET_REPO_PATH = '/xrpc/com.atproto.sync.getRepo'
-
-export const buildRateLimitsConfig = (
-  rateLimits: RateLimitsConfig,
-  redisScratch?: Redis,
-): RateLimitDescriptions | undefined => {
-  if (!rateLimits.enabled) return undefined
-
-  return {
-    creator: redisScratch
-      ? (opts) => new RedisRateLimiter(redisScratch, opts)
-      : (opts) => new MemoryRateLimiter(opts),
-    bypass: ({ req }) => {
-      const { bypassKey, bypassIps } = rateLimits
-      if (bypassKey && bypassKey === req.headers['x-ratelimit-bypass']) {
-        return true
-      }
-      if (bypassIps && bypassIps.includes(req.ip)) {
-        return true
-      }
-      return false
-    },
-    global: [
-      {
-        name: 'global-ip',
-        durationMs: 5 * MINUTE,
-        points: 3000,
-        // getRepo can be a high-volume sync path, so it has its own endpoint
-        // limit and should not consume the shared global read bucket.
-        calcKey: ({ req }) => {
-          if (req.path === SYNC_GET_REPO_PATH) {
-            return null
-          }
-          return req.ip
-        },
-      },
-    ],
-    shared: [
-      {
-        name: 'repo-write-hour',
-        durationMs: HOUR,
-        points: 5000, // creates=3, puts=2, deletes=1
-      },
-      {
-        name: 'repo-write-day',
-        durationMs: DAY,
-        points: 35000, // creates=3, puts=2, deletes=1
-      },
-    ],
-  }
-}

package/src/read-after-write/index.ts

@@ -1,3 +0,0 @@
-export * from './types.js'
-export * from './util.js'
-export * from './viewer.js'

package/src/read-after-write/types.ts

@@ -1,35 +0,0 @@
-import { Cid } from '@atproto/lex-data'
-import { AtUri, DatetimeString } from '@atproto/syntax'
-import { app } from '../lexicons/index.js'
-import { LocalViewer } from './viewer.js'
-
-export type LocalRecords = {
-  count: number
-  profile: RecordDescript<app.bsky.actor.profile.Main> | null
-  posts: RecordDescript<app.bsky.feed.post.Main>[]
-}
-
-export type RecordDescript<T> = {
-  uri: AtUri
-  cid: Cid
-  indexedAt: DatetimeString
-  record: T
-}
-
-export type ApiRes<T> = {
-  headers: Record<string, string | undefined>
-  data: T
-}
-
-export type MungeFn<T> = (
-  localViewer: LocalViewer,
-  original: T,
-  local: LocalRecords,
-  requester: string,
-) => Promise<T>
-
-export type HandlerResponse<T> = {
-  encoding: 'application/json'
-  body: T
-  headers?: Record<string, string>
-}

package/src/read-after-write/util.ts

@@ -1,101 +0,0 @@
-import express from 'express'
-import { LexValue, l } from '@atproto/lex'
-import { lexParse } from '@atproto/lex-json'
-import {
-  HandlerPipeThrough,
-  HandlerPipeThroughBuffer,
-} from '@atproto/xrpc-server'
-import { AppContext } from '../context.js'
-import { readStickyLogger as log } from '../logger.js'
-import {
-  asPipeThroughBuffer,
-  isJsonContentType,
-  pipethrough,
-} from '../pipethrough.js'
-import { HandlerResponse, LocalRecords, MungeFn } from './types.js'
-
-export const getLocalLag = (local: LocalRecords): number | undefined => {
-  let oldest: string | undefined = local.profile?.indexedAt
-  for (const post of local.posts) {
-    if (!oldest || post.indexedAt < oldest) {
-      oldest = post.indexedAt
-    }
-  }
-  if (!oldest) return undefined
-  return Date.now() - new Date(oldest).getTime()
-}
-
-export const pipethroughReadAfterWrite = async <
-  M extends (l.Query | l.Procedure) & {
-    output: l.Payload<`application/json`, l.Schema<LexValue>>
-  },
->(
-  ctx: AppContext,
-  reqCtx: { req: express.Request; auth: { credentials: { did: string } } },
-  ns: l.Main<M>,
-  munge: MungeFn<l.InferMethodOutputBody<M>>,
-): Promise<
-  HandlerResponse<l.InferMethodOutputBody<M>> | HandlerPipeThrough
-> => {
-  const { req, auth } = reqCtx
-  const requester = auth.credentials.did
-  const method = l.getMain(ns)
-
-  const streamRes = await pipethrough(ctx, req, { iss: requester })
-
-  const rev = streamRes.headers['atproto-repo-rev']
-  if (!rev) return streamRes
-
-  if (isJsonContentType(streamRes.headers['content-type']) === false) {
-    // content-type is present but not JSON, we can't munge this
-    return streamRes
-  }
-
-  // if the munging fails, we can't return the original response because the
-  // stream will already have been read. If we end-up buffering the response,
-  // we'll return the buffered response in case of an error.
-  let bufferRes: HandlerPipeThroughBuffer | undefined
-
-  try {
-    return await ctx.actorStore.read(requester, async (store) => {
-      const local = await store.record.getRecordsSinceRev(rev)
-      if (local.count === 0) return streamRes
-
-      const { buffer } = (bufferRes = await asPipeThroughBuffer(streamRes))
-
-      const lex = lexParse(buffer.toString('utf8'), { strict: false })
-
-      const result = method.output.schema.safeValidate(lex, { strict: false })
-
-      // We won't perform munging with invalid upstream data
-      if (!result.success) return bufferRes
-
-      const parsedRes = result.value as l.InferMethodOutputBody<M, never>
-
-      const localViewer = ctx.localViewer(store)
-
-      const data = await munge(localViewer, parsedRes, local, requester)
-      return formatMungedResponse(data, getLocalLag(local))
-    })
-  } catch (err) {
-    // The error occurred while reading the stream, this is non-recoverable
-    if (!bufferRes && !streamRes.stream.readable) throw err
-
-    log.warn({ err, requester }, 'error in read after write munge')
-    return bufferRes ?? streamRes
-  }
-}
-
-export const formatMungedResponse = <T>(
-  body: T,
-  lag?: number,
-): HandlerResponse<T> => ({
-  encoding: 'application/json',
-  body,
-  headers:
-    lag !== undefined
-      ? {
-          'Atproto-Upstream-Lag': lag.toString(10),
-        }
-      : undefined,
-})

package/src/read-after-write/viewer.ts

@@ -1,290 +0,0 @@
-import { LexMap, UriString, getBlobCidString } from '@atproto/lex'
-import { AtUri, DidString, HandleString, INVALID_HANDLE } from '@atproto/syntax'
-import { createServiceAuthHeaders } from '@atproto/xrpc-server'
-import { AccountManager } from '../account-manager/account-manager.js'
-import { ActorStoreReader } from '../actor-store/actor-store-reader.js'
-import { BskyAppView } from '../bsky-app-view.js'
-import { ImageUrlBuilder } from '../image/image-url-builder.js'
-import { app } from '../lexicons/index.js'
-import { LocalRecords, RecordDescript } from './types.js'
-
-type CommonSignedUris = 'avatar' | 'banner' | 'feed_thumbnail' | 'feed_fullsize'
-
-export type LocalViewerCreator = (
-  actorStoreReader: ActorStoreReader,
-) => LocalViewer
-
-export class LocalViewer {
-  constructor(
-    public readonly actorStoreReader: ActorStoreReader,
-    public readonly accountManager: AccountManager,
-    public readonly imageUrlBuilder: ImageUrlBuilder,
-    public readonly bskyAppView?: BskyAppView,
-  ) {}
-
-  get did() {
-    return this.actorStoreReader.did as DidString
-  }
-
-  static creator(
-    accountManager: AccountManager,
-    imageUrlBuilder: ImageUrlBuilder,
-    bskyAppView?: BskyAppView,
-  ): LocalViewerCreator {
-    return (actorStore) =>
-      new LocalViewer(actorStore, accountManager, imageUrlBuilder, bskyAppView)
-  }
-
-  getImageUrl(pattern: CommonSignedUris, cid: string) {
-    return this.imageUrlBuilder.build(pattern, this.did, cid) as UriString
-  }
-
-  async serviceAuthHeaders(did: string, lxm: string) {
-    if (!this.bskyAppView) {
-      throw new Error('Could not find bsky appview did')
-    }
-    const keypair = await this.actorStoreReader.keypair()
-
-    return createServiceAuthHeaders({
-      iss: did,
-      aud: this.bskyAppView.did,
-      lxm,
-      keypair,
-    })
-  }
-
-  async getRecordsSinceRev(rev: string): Promise<LocalRecords> {
-    return this.actorStoreReader.record.getRecordsSinceRev(rev)
-  }
-
-  async getProfileBasic(): Promise<app.bsky.actor.defs.ProfileViewBasic | null> {
-    const [profileRes, accountRes] = await Promise.all([
-      this.actorStoreReader.record.getProfileRecord(),
-      this.accountManager.getAccount(this.did),
-    ])
-
-    if (!accountRes) return null
-
-    return {
-      did: this.did,
-      handle: (accountRes.handle ?? INVALID_HANDLE) as HandleString,
-      displayName: profileRes?.displayName,
-      avatar: profileRes?.avatar
-        ? this.getImageUrl('avatar', getBlobCidString(profileRes.avatar))
-        : undefined,
-    }
-  }
-
-  async formatAndInsertPostsInFeed(
-    feed: app.bsky.feed.defs.FeedViewPost[],
-    posts: RecordDescript<app.bsky.feed.post.Main>[],
-  ): Promise<app.bsky.feed.defs.FeedViewPost[]> {
-    if (posts.length === 0) {
-      return feed
-    }
-    const lastTime = feed.at(-1)?.post.indexedAt ?? new Date(0).toISOString()
-    const inFeed = posts.filter((p) => p.indexedAt > lastTime)
-    const newestToOldest = inFeed.reverse()
-    const maybeFormatted = await Promise.all(
-      newestToOldest.map((p) => this.getPost(p)),
-    )
-    const formatted = maybeFormatted.filter(
-      (p) => p !== null,
-    ) as app.bsky.feed.defs.PostView[]
-    for (const post of formatted) {
-      const idx = feed.findIndex((fi) => fi.post.indexedAt < post.indexedAt)
-      if (idx >= 0) {
-        feed.splice(idx, 0, { post })
-      } else {
-        feed.push({ post })
-      }
-    }
-    return feed
-  }
-
-  async getPost(
-    descript: RecordDescript<app.bsky.feed.post.Main>,
-  ): Promise<app.bsky.feed.defs.PostView | null> {
-    const { uri, cid, indexedAt, record } = descript
-    const author = await this.getProfileBasic()
-    if (!author) return null
-    const embed = record.embed ? await this.formatPostEmbed(record) : undefined
-    return {
-      uri: uri.toString(),
-      cid: cid.toString(),
-      likeCount: 0, // counts presumed to be 0 directly after post creation
-      replyCount: 0,
-      repostCount: 0,
-      quoteCount: 0,
-      author,
-      record: record as LexMap,
-      embed,
-      indexedAt,
-    }
-  }
-
-  async formatPostEmbed(post: app.bsky.feed.post.Main) {
-    const embed = post.embed
-    if (!embed) return undefined
-    if (app.bsky.embed.images.$isTypeOf(embed)) {
-      return this.formatImageEmbed(embed)
-    } else if (app.bsky.embed.external.$isTypeOf(embed)) {
-      return this.formatExternalEmbed(embed)
-    } else if (app.bsky.embed.record.$isTypeOf(embed)) {
-      return this.formatRecordEmbed(embed)
-    } else if (app.bsky.embed.recordWithMedia.$isTypeOf(embed)) {
-      return this.formatRecordWithMediaEmbed(embed)
-    } else {
-      return undefined
-    }
-  }
-
-  formatImageEmbed(embed: app.bsky.embed.images.Main) {
-    const images = embed.images.map(
-      (img): app.bsky.embed.images.ViewImage => ({
-        thumb: this.getImageUrl('feed_thumbnail', getBlobCidString(img.image)),
-        fullsize: this.getImageUrl(
-          'feed_fullsize',
-          getBlobCidString(img.image),
-        ),
-        aspectRatio: img.aspectRatio,
-        alt: img.alt,
-      }),
-    )
-    return app.bsky.embed.images.view.$build({ images })
-  }
-
-  formatExternalEmbed(embed: app.bsky.embed.external.Main) {
-    const { uri, title, description, thumb } = embed.external
-    return app.bsky.embed.external.view.$build({
-      external: {
-        uri,
-        title,
-        description,
-        thumb: thumb
-          ? this.getImageUrl('feed_thumbnail', getBlobCidString(thumb))
-          : undefined,
-      },
-    })
-  }
-
-  async formatRecordEmbed(embed: app.bsky.embed.record.Main) {
-    const view = await this.formatRecordEmbedInternal(embed)
-    return app.bsky.embed.record.view.$build({
-      record:
-        view ??
-        app.bsky.embed.record.viewNotFound.$build({
-          uri: embed.record.uri,
-          notFound: true,
-        }),
-    })
-  }
-
-  private async formatRecordEmbedInternal(embed: app.bsky.embed.record.Main) {
-    if (!this.bskyAppView) {
-      return undefined
-    }
-    const collection = new AtUri(embed.record.uri).collection
-    if (collection === app.bsky.feed.post.$type) {
-      const { headers } = await this.serviceAuthHeaders(
-        this.did,
-        app.bsky.feed.getPosts.$lxm,
-      )
-      const data = await this.bskyAppView.client.call(
-        app.bsky.feed.getPosts,
-        { uris: [embed.record.uri] },
-        { headers },
-      )
-      const post = data.posts[0]
-      if (!post) return undefined
-
-      return app.bsky.embed.record.viewRecord.$build({
-        uri: post.uri,
-        cid: post.cid,
-        author: post.author,
-        value: post.record,
-        labels: post.labels,
-        embeds: post.embed ? [post.embed] : undefined,
-        indexedAt: post.indexedAt,
-      })
-    } else if (collection === app.bsky.feed.generator.$type) {
-      const { headers } = await this.serviceAuthHeaders(
-        this.did,
-        app.bsky.feed.getFeedGenerator.$lxm,
-      )
-      const data = await this.bskyAppView.client.call(
-        app.bsky.feed.getFeedGenerator,
-        { feed: embed.record.uri },
-        { headers },
-      )
-      return app.bsky.feed.defs.generatorView.$build(data.view)
-    } else if (collection === app.bsky.graph.list.$type) {
-      const { headers } = await this.serviceAuthHeaders(
-        this.did,
-        app.bsky.graph.getList.$lxm,
-      )
-      const data = await this.bskyAppView.client.call(
-        app.bsky.graph.getList,
-        { list: embed.record.uri },
-        { headers },
-      )
-      return app.bsky.graph.defs.listView.$build(data.list)
-    }
-    return undefined
-  }
-
-  async formatRecordWithMediaEmbed(embed: app.bsky.embed.recordWithMedia.Main) {
-    const media = app.bsky.embed.images.$isTypeOf(embed.media)
-      ? this.formatImageEmbed(embed.media)
-      : app.bsky.embed.external.$isTypeOf(embed.media)
-        ? this.formatExternalEmbed(embed.media)
-        : null
-
-    if (!media) return undefined
-
-    const record = await this.formatRecordEmbed(embed.record)
-    return app.bsky.embed.recordWithMedia.view.$build({
-      record,
-      media,
-    })
-  }
-
-  updateProfileViewBasic<
-    T extends
-      | app.bsky.actor.defs.ProfileViewDetailed
-      | app.bsky.actor.defs.ProfileViewBasic
-      | app.bsky.actor.defs.ProfileView,
-  >(view: T, record: app.bsky.actor.profile.Main): T {
-    return {
-      ...view,
-      displayName: record.displayName,
-      avatar: record.avatar
-        ? this.getImageUrl('avatar', getBlobCidString(record.avatar))
-        : undefined,
-    }
-  }
-
-  updateProfileView<
-    T extends
-      | app.bsky.actor.defs.ProfileViewDetailed
-      | app.bsky.actor.defs.ProfileViewBasic
-      | app.bsky.actor.defs.ProfileView,
-  >(view: T, record: app.bsky.actor.profile.Main): T {
-    return {
-      ...this.updateProfileViewBasic(view, record),
-      description: record.description,
-    }
-  }
-
-  updateProfileDetailed<T extends app.bsky.actor.defs.ProfileViewDetailed>(
-    view: T,
-    record: app.bsky.actor.profile.Main,
-  ): T {
-    return {
-      ...this.updateProfileView(view, record),
-      banner: record.banner
-        ? this.getImageUrl('banner', getBlobCidString(record.banner))
-        : undefined,
-    }
-  }
-}

package/src/redis.ts

@@ -1,25 +0,0 @@
-import assert from 'node:assert'
-import { Redis } from 'ioredis'
-import { redisLogger } from './logger.js'
-
-export const getRedisClient = (host: string, password?: string): Redis => {
-  const redisAddr = redisAddressParts(host)
-  const redis = new Redis({
-    ...redisAddr,
-    password,
-  })
-  redis.on('error', (err) => {
-    redisLogger.error({ host, err }, 'redis error')
-  })
-  return redis
-}
-
-export const redisAddressParts = (
-  addr: string,
-  defaultPort = 6379,
-): { host: string; port: number } => {
-  const [host, portStr, ...others] = addr.split(':')
-  const port = portStr ? parseInt(portStr, 10) : defaultPort
-  assert(host && !isNaN(port) && !others.length, `invalid address: ${addr}`)
-  return { host, port }
-}

package/src/repo/index.ts

@@ -1,2 +0,0 @@
-export * from './prepare.js'
-export * from './types.js'