@atproto/bsky 0.0.103 → 0.0.105

package/src/context.ts

@@ -17,6 +17,7 @@ import {
   parseLabelerHeader,
 } from './util'
 import { httpLogger as log } from './logger'
+import { Dispatcher } from 'undici'
 
 export class AppContext {
   constructor(
@@ -34,6 +35,7 @@ export class AppContext {
       courierClient: CourierClient | undefined
       authVerifier: AuthVerifier
       featureGates: FeatureGates
+      blobDispatcher: Dispatcher
     },
   ) {}
 
@@ -93,6 +95,10 @@ export class AppContext {
     return this.opts.featureGates
   }
 
+  get blobDispatcher(): Dispatcher {
+    return this.opts.blobDispatcher
+  }
+
   reqLabelers(req: express.Request): ParsedLabelers {
     const val = req.header('atproto-accept-labelers')
     let parsed: ParsedLabelers | null

package/src/data-plane/server/indexing/index.ts

@@ -31,7 +31,7 @@ import * as Labeler from './plugins/labeler'
 import * as ChatDeclaration from './plugins/chat-declaration'
 import RecordProcessor from './processor'
 import { subLogger } from '../../../logger'
-import { retryHttp } from '../../../util/retry'
+import { retryXrpc } from '../../../util/retry'
 import { BackgroundQueue } from '../background'
 
 export class IndexingService {
@@ -165,7 +165,7 @@ export class IndexingService {
     )
     const { api } = new AtpAgent({ service: pds })
 
-    const { data: car } = await retryHttp(() =>
+    const { data: car } = await retryXrpc(() =>
       api.com.atproto.sync.getRepo({ did }),
     )
     const { root, blocks } = await readCarWithRoot(car)
@@ -287,7 +287,7 @@ export class IndexingService {
     if (!pds) return false
     const { api } = new AtpAgent({ service: pds })
     try {
-      await retryHttp(() => api.com.atproto.sync.getLatestCommit({ did }))
+      await retryXrpc(() => api.com.atproto.sync.getLatestCommit({ did }))
       return true
     } catch (err) {
       if (err instanceof ComAtprotoSyncGetLatestCommit.RepoNotFoundError) {

package/src/image/server.ts

@@ -1,136 +1,170 @@
-import fs from 'fs/promises'
-import fsSync from 'fs'
-import os from 'os'
-import path from 'path'
-import { Readable } from 'stream'
-import axios, { AxiosError } from 'axios'
-import express, {
-  Request,
-  Response,
-  Express,
-  ErrorRequestHandler,
-  NextFunction,
-} from 'express'
-import createError, { isHttpError } from 'http-errors'
-import { BlobNotFoundError } from '@atproto/repo'
 import {
   cloneStream,
-  forwardStreamErrors,
+  createDecoders,
   isErrnoException,
+  VerifyCidError,
+  VerifyCidTransform,
 } from '@atproto/common'
-import { BadPathError, ImageUriBuilder } from './uri'
+import { BlobNotFoundError } from '@atproto/repo'
+import createError, { isHttpError } from 'http-errors'
+import fsSync from 'node:fs'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { Duplex, Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+
+import { streamBlob, StreamBlobOptions } from '../api/blob-resolver'
+import AppContext from '../context'
+import { Middleware, responseSignal } from '../util/http'
 import log from './logger'
-import { resize } from './sharp'
-import { formatsToMimes, Options } from './util'
-import { retryHttp } from '../util/retry'
-import { ServerConfig } from '../config'
-
-export class ImageProcessingServer {
-  app: Express = express()
-  uriBuilder: ImageUriBuilder
-
-  constructor(
-    public cfg: ServerConfig,
-    public cache: BlobCache,
-  ) {
-    this.uriBuilder = new ImageUriBuilder('')
-    this.app.get('*', this.handler.bind(this))
-    this.app.use(errorMiddleware)
+import { createImageProcessor, createImageUpscaler } from './sharp'
+import { BadPathError, ImageUriBuilder } from './uri'
+import { formatsToMimes, Options, SharpInfo } from './util'
+
+export function createMiddleware(
+  ctx: AppContext,
+  { prefix = '/' }: { prefix?: string } = {},
+): Middleware {
+  if (!prefix.startsWith('/') || !prefix.endsWith('/')) {
+    throw new TypeError('Prefix must start and end with a slash')
+  }
+
+  // If there is a CDN, we don't need to serve images
+  if (ctx.cfg.cdnUrl) {
+    return (req, res, next) => next()
   }
 
-  async handler(req: Request, res: Response, next: NextFunction) {
+  const cache = new BlobDiskCache(ctx.cfg.blobCacheLocation)
+
+  return async (req, res, next) => {
+    if (res.destroyed) return
+    if (req.method !== 'GET' && req.method !== 'HEAD') return next()
+    if (!req.url?.startsWith(prefix)) return next()
+    const { 0: path, 1: _search } = req.url.slice(prefix.length - 1).split('?')
+    if (!path.startsWith('/') || path === '/') return next()
+
     try {
-      const path = req.path
       const options = ImageUriBuilder.getOptions(path)
-      const cacheKey = [
-        options.did,
-        options.cid.toString(),
-        options.preset,
-      ].join('::')
+
+      const cacheKey = [options.did, options.cid, options.preset].join('::')
 
       // Cached flow
 
       try {
-        const cachedImage = await this.cache.get(cacheKey)
+        const cachedImage = await cache.get(cacheKey)
         res.statusCode = 200
         res.setHeader('x-cache', 'hit')
         res.setHeader('content-type', getMime(options.format))
         res.setHeader('cache-control', `public, max-age=31536000`) // 1 year
         res.setHeader('content-length', cachedImage.size)
-        forwardStreamErrors(cachedImage, res)
-        return cachedImage.pipe(res)
+        await pipeline(cachedImage, res)
+        return
       } catch (err) {
-        // Ignore BlobNotFoundError and move on to non-cached flow
-        if (!(err instanceof BlobNotFoundError)) throw err
+        if (!(err instanceof BlobNotFoundError)) {
+          log.error({ cacheKey, err }, 'failed to serve cached image')
+        }
+
+        if (res.headersSent || res.destroyed) {
+          res.destroy()
+          return // nothing we can do...
+        } else {
+          // Ignore and move on to non-cached flow.
+          res.removeHeader('x-cache')
+          res.removeHeader('content-type')
+          res.removeHeader('cache-control')
+          res.removeHeader('content-length')
+        }
       }
 
       // Non-cached flow
 
-      const { localUrl } = this.cfg
-      const did = options.did
-      const cidStr = options.cid.toString()
-
-      const blobResult = await retryHttp(() =>
-        getBlob({ baseUrl: localUrl, did, cid: cidStr }),
-      )
-
-      const imageStream: Readable = blobResult.data
-      const processedImage = await resize(imageStream, options)
-
-      // Cache in the background
-      this.cache
-        .put(cacheKey, cloneStream(processedImage))
-        .catch((err) => log.error(err, 'failed to cache image'))
-      // Respond
-      res.statusCode = 200
-      res.setHeader('x-cache', 'miss')
-      res.setHeader('content-type', getMime(options.format))
-      res.setHeader('cache-control', `public, max-age=31536000`) // 1 year
-      forwardStreamErrors(processedImage, res)
-      return (
-        processedImage
-          // @NOTE sharp does emit this in time to be set as a header
-          .once('info', (info) => res.setHeader('content-length', info.size))
-          .pipe(res)
-      )
-    } catch (err: unknown) {
-      if (err instanceof BadPathError) {
-        return next(createError(400, err))
+      const streamOptions: StreamBlobOptions = {
+        did: options.did,
+        cid: options.cid,
+        signal: responseSignal(res),
       }
-      if (err instanceof AxiosError) {
-        if (err.code === AxiosError.ETIMEDOUT) {
-          return next(createError(504)) // Gateway timeout
-        }
-        if (!err.response || err.response.status >= 500) {
-          return next(createError(502))
+
+      await streamBlob(ctx, streamOptions, (upstream, { did, cid, url }) => {
+        // Definitely not an image ? Let's fail right away.
+        if (isImageMime(upstream.headers['content-type']) === false) {
+          throw createError(400, 'Not an image')
         }
-        if (err.response.status === 400) {
-          return next(createError(400))
+
+        // Let's transform (decompress, verify CID, upscale), process and respond
+
+        const transforms: Duplex[] = [
+          ...createDecoders(upstream.headers['content-encoding']),
+          new VerifyCidTransform(cid),
+          createImageUpscaler(options),
+        ]
+        const processor = createImageProcessor(options)
+
+        // Cache in the background
+        cache
+          .put(cacheKey, cloneStream(processor))
+          .catch((err) => log.error(err, 'failed to cache image'))
+
+        res.statusCode = 200
+        res.setHeader('cache-control', `public, max-age=31536000`) // 1 year
+        res.setHeader('x-cache', 'miss')
+        processor.once('info', ({ size, format }: SharpInfo) => {
+          const type = formatsToMimes.get(format) || 'application/octet-stream'
+
+          // @NOTE sharp does emit this in time to be set as a header
+          res.setHeader('content-length', size)
+          res.setHeader('content-type', type)
+        })
+
+        const streams = [...transforms, processor, res]
+        void pipeline(streams).catch((err: unknown) => {
+          log.warn(
+            { err, did, cid: cid.toString(), pds: url.origin },
+            'blob resolution failed during transmission',
+          )
+        })
+
+        return streams[0]!
+      })
+    } catch (err) {
+      if (res.headersSent || res.destroyed) {
+        res.destroy()
+      } else {
+        res.removeHeader('content-type')
+        res.removeHeader('content-length')
+        res.removeHeader('cache-control')
+        res.removeHeader('x-cache')
+
+        if (err instanceof BadPathError) {
+          next(createError(400, err))
+        } else if (err instanceof VerifyCidError) {
+          next(createError(404, 'Blob not found', err))
+        } else if (isHttpError(err)) {
+          next(err)
+        } else {
+          next(createError(502, 'Upstream Error', { cause: err }))
         }
-        return next(createError(404, 'Image not found'))
       }
-      return next(err)
     }
   }
 }
 
-const errorMiddleware: ErrorRequestHandler = function (err, _req, res, next) {
-  if (isHttpError(err)) {
-    log.error(err, `error: ${err.message}`)
-  } else {
-    log.error(err, 'unhandled exception')
+function isImageMime(
+  contentType: string | string[] | undefined,
+): undefined | boolean {
+  if (contentType == null || contentType === 'application/octet-stream') {
+    return undefined // maybe
   }
-  if (res.headersSent) {
-    return next(err)
+  if (Array.isArray(contentType)) {
+    if (contentType.length === 0) return undefined // should never happen
+    if (contentType.length === 1) return isImageMime(contentType[0])
+    return contentType.every(isImageMime) // Should we throw a 502 here?
   }
-  const httpError = createError(err)
-  return res.status(httpError.status).json({
-    message: httpError.expose ? httpError.message : 'Internal Server Error',
-  })
+  return contentType.startsWith('image/')
 }
 
 function getMime(format: Options['format']) {
-  const mime = formatsToMimes[format]
+  const mime = formatsToMimes.get(format)
   if (!mime) throw new Error('Unknown format')
   return mime
 }
@@ -193,13 +227,3 @@ export class BlobDiskCache implements BlobCache {
     await fs.rm(this.tempDir, { recursive: true, force: true })
   }
 }
-
-function getBlob(opts: { baseUrl: string; did: string; cid: string }) {
-  const { baseUrl, did, cid } = opts
-  const enc = encodeURIComponent
-  return axios.get(`${baseUrl}/blob/${enc(did)}/${enc(cid)}`, {
-    decompress: true,
-    responseType: 'stream',
-    timeout: 2000, // 2sec of inactivity on the connection
-  })
-}

package/src/image/sharp.ts

@@ -1,87 +1,83 @@
-import { Readable } from 'stream'
-import { pipeline } from 'stream/promises'
+import { errHasMsg } from '@atproto/common'
+import { PassThrough, Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
 import sharp from 'sharp'
-import { errHasMsg, forwardStreamErrors } from '@atproto/common'
 import { formatsToMimes, ImageInfo, Options } from './util'
 
 export type { Options }
 
-export async function resize(
-  stream: Readable,
-  options: Options,
-): Promise<Readable> {
-  const { height, width, min = false, fit = 'cover', format, quality } = options
-
-  let processor = sharp()
-
-  // Scale up to hit any specified minimum size
-  if (typeof min !== 'boolean') {
-    const upsizeProcessor = sharp().resize({
-      fit: 'outside',
-      width: min.width,
-      height: min.height,
-      withoutReduction: true,
-      withoutEnlargement: false,
-    })
-    forwardStreamErrors(stream, upsizeProcessor)
-    stream = stream.pipe(upsizeProcessor)
-  }
+/**
+ * Scale up to hit any specified minimum size
+ */
+export function createImageUpscaler({ min = false }: Options) {
+  // Due to the way sharp works, up-scaling must happen in a separate processor
+  // than down-scaling.
+  return typeof min !== 'boolean'
+    ? sharp().resize({
+        fit: 'outside',
+        width: min.width,
+        height: min.height,
+        withoutReduction: true,
+        withoutEnlargement: false,
+      })
+    : new PassThrough()
+}
 
-  // Scale down (or possibly up if min is true) to desired size
-  processor = processor.resize({
+/**
+ * Scale down (or possibly up if min is true) to desired size, then compress
+ * to the desired format.
+ */
+export function createImageProcessor({
+  height,
+  width,
+  min = false,
+  fit = 'cover',
+  format,
+  quality = 100,
+}: Options) {
+  const processor = sharp().resize({
     fit,
     width,
     height,
     withoutEnlargement: min !== true,
   })
 
-  // Output to specified format
   if (format === 'jpeg') {
-    processor = processor.jpeg({ quality: quality ?? 100 })
+    return processor.jpeg({ quality })
   } else if (format === 'png') {
-    processor = processor.png({ quality: quality ?? 100 })
+    return processor.png({ quality })
   } else {
-    const exhaustiveCheck: never = format
-    throw new Error(`Unhandled case: ${exhaustiveCheck}`)
+    throw new Error(`Unhandled case: ${format}`)
   }
-
-  forwardStreamErrors(stream, processor)
-  return stream.pipe(processor)
 }
 
 export async function maybeGetInfo(
   stream: Readable,
 ): Promise<ImageInfo | null> {
-  let metadata: sharp.Metadata
   try {
     const processor = sharp()
-    const [result] = await Promise.all([
+
+    const [{ size, height, width, format }] = await Promise.all([
       processor.metadata(),
       pipeline(stream, processor), // Handles error propagation
     ])
-    metadata = result
+
+    if (size == null || height == null || width == null || format == null) {
+      return null
+    }
+
+    return {
+      height,
+      width,
+      size,
+      mime: formatsToMimes.get(format) ?? 'unknown',
+    }
   } catch (err) {
     if (errHasMsg(err, 'Input buffer contains unsupported image format')) {
       return null
     }
     throw err
   }
-  const { size, height, width, format } = metadata
-  if (
-    size === undefined ||
-    height === undefined ||
-    width === undefined ||
-    format === undefined
-  ) {
-    return null
-  }
-
-  return {
-    height,
-    width,
-    size,
-    mime: formatsToMimes[format] ?? ('unknown' as const),
-  }
 }
 
 export async function getInfo(stream: Readable): Promise<ImageInfo> {

package/src/image/util.ts

@@ -1,4 +1,6 @@
-import { FormatEnum } from 'sharp'
+import { FormatEnum, OutputInfo } from 'sharp'
+
+export type ImageMime = `image/${string}`
 
 export type Options = Dimensions & {
   format: 'jpeg' | 'png'
@@ -15,18 +17,24 @@ export type Options = Dimensions & {
 
 export type ImageInfo = Dimensions & {
   size: number
-  mime: `image/${string}` | 'unknown'
+  mime: ImageMime | 'unknown'
 }
 
 export type Dimensions = { height: number; width: number }
 
-export const formatsToMimes: { [s in keyof FormatEnum]?: `image/${string}` } = {
-  jpg: 'image/jpeg',
-  jpeg: 'image/jpeg',
-  png: 'image/png',
-  gif: 'image/gif',
-  svg: 'image/svg+xml',
-  tif: 'image/tiff',
-  tiff: 'image/tiff',
-  webp: 'image/webp',
-}
+export const formatsToMimes = new Map<keyof FormatEnum, ImageMime>([
+  ['jpg', 'image/jpeg'],
+  ['jpeg', 'image/jpeg'],
+  ['png', 'image/png'],
+  ['gif', 'image/gif'],
+  ['svg', 'image/svg+xml'],
+  ['tif', 'image/tiff'],
+  ['tiff', 'image/tiff'],
+  ['webp', 'image/webp'],
+  ['avif', 'image/avif'],
+  ['heif', 'image/heif'],
+  ['jp2', 'image/jp2'],
+  ['jxl', 'image/jxl'],
+])
+
+export type SharpInfo = OutputInfo & { format: keyof FormatEnum }

package/src/index.ts

@@ -8,15 +8,15 @@ import compression from 'compression'
 import { AtpAgent } from '@atproto/api'
 import { IdResolver } from '@atproto/identity'
 import { DAY, SECOND } from '@atproto/common'
+import { Keypair } from '@atproto/crypto'
 import API, { health, wellKnown, blobResolver } from './api'
 import * as error from './error'
 import { loggerMiddleware } from './logger'
 import { ServerConfig } from './config'
 import { createServer } from './lexicon'
 import { ImageUriBuilder } from './image/uri'
-import { BlobDiskCache, ImageProcessingServer } from './image/server'
+import * as imageServer from './image/server'
 import AppContext from './context'
-import { Keypair } from '@atproto/crypto'
 import { createDataPlaneClient } from './data-plane/client'
 import { Hydrator } from './hydration/hydrator'
 import { Views } from './views'
@@ -25,6 +25,7 @@ import { authWithApiKey as bsyncAuth, createBsyncClient } from './bsync'
 import { authWithApiKey as courierAuth, createCourierClient } from './courier'
 import { FeatureGates } from './feature-gates'
 import { VideoUriBuilder } from './views/util'
+import { createBlobDispatcher } from './api/blob-dispatcher'
 
 export * from './data-plane'
 export type { ServerConfigValues } from './config'
@@ -73,15 +74,6 @@ export class BskyAppView {
         `${config.publicUrl}/vid/%s/%s/thumbnail.jpg`,
     })
 
-    let imgProcessingServer: ImageProcessingServer | undefined
-    if (!config.cdnUrl) {
-      const imgProcessingCache = new BlobDiskCache(config.blobCacheLocation)
-      imgProcessingServer = new ImageProcessingServer(
-        config,
-        imgProcessingCache,
-      )
-    }
-
     const searchAgent = config.searchUrl
       ? new AtpAgent({ service: config.searchUrl })
       : undefined
@@ -151,6 +143,8 @@ export class BskyAppView {
       env: config.statsigEnv,
     })
 
+    const blobDispatcher = createBlobDispatcher(config)
+
     const ctx = new AppContext({
       cfg: config,
       dataplane,
@@ -165,6 +159,7 @@ export class BskyAppView {
       courierClient,
       authVerifier,
       featureGates,
+      blobDispatcher,
     })
 
     let server = createServer({
@@ -180,10 +175,8 @@ export class BskyAppView {
 
     app.use(health.createRouter(ctx))
     app.use(wellKnown.createRouter(ctx))
-    app.use(blobResolver.createRouter(ctx))
-    if (imgProcessingServer) {
-      app.use('/img', imgProcessingServer.app)
-    }
+    app.use(blobResolver.createMiddleware(ctx))
+    app.use(imageServer.createMiddleware(ctx, { prefix: '/img/' }))
     app.use(server.xrpc.router)
     app.use(error.handler)
 

package/src/util/http.ts

@@ -0,0 +1,41 @@
+import createHttpError from 'http-errors'
+import { IncomingMessage, ServerResponse } from 'node:http'
+import { IncomingHttpHeaders } from 'undici/types/header'
+
+type NextFunction = (err?: unknown) => void
+
+export type Middleware = (
+  req: IncomingMessage,
+  res: ServerResponse,
+  next: NextFunction,
+) => void
+
+export type ResponseData = { statusCode: number; headers: IncomingHttpHeaders }
+
+const RESPONSE_HEADERS_TO_PROXY = new Set([
+  'content-type',
+  'content-length',
+  'content-encoding',
+  'content-language',
+  'cache-control',
+  'last-modified',
+  'etag',
+  'expires',
+  'retry-after',
+  'vary', // Might vary based on "accept" headers
+] as const satisfies (keyof IncomingHttpHeaders)[])
+
+export function proxyResponseHeaders(data: ResponseData, res: ServerResponse) {
+  res.statusCode = data.statusCode >= 500 ? 502 : data.statusCode
+  for (const name of RESPONSE_HEADERS_TO_PROXY) {
+    const val = data.headers[name]
+    if (val) res.setHeader(name, val)
+  }
+}
+
+export function responseSignal(res: ServerResponse): AbortSignal {
+  if (res.destroyed) throw createHttpError(499, 'Client Disconnected')
+  const controller = new AbortController()
+  res.once('close', () => controller.abort())
+  return controller.signal
+}

package/src/util/retry.ts

@@ -1,38 +1,14 @@
-import { AxiosError } from 'axios'
-import { XRPCError, ResponseType } from '@atproto/xrpc'
-import { RetryOptions, retry } from '@atproto/common'
-import { Code, ConnectError } from '@connectrpc/connect'
+import { createRetryable } from '@atproto/common'
+import { ResponseType, XRPCError } from '@atproto/xrpc'
 
-export async function retryHttp<T>(
-  fn: () => Promise<T>,
-  opts: RetryOptions = {},
-): Promise<T> {
-  return retry(fn, { retryable: retryableHttp, ...opts })
-}
+export const RETRYABLE_HTTP_STATUS_CODES = new Set([
+  408, 425, 429, 500, 502, 503, 504, 522, 524,
+])
 
-export function retryableHttp(err: unknown) {
+export const retryXrpc = createRetryable((err: unknown) => {
   if (err instanceof XRPCError) {
     if (err.status === ResponseType.Unknown) return true
-    return retryableHttpStatusCodes.has(err.status)
-  }
-  if (err instanceof AxiosError) {
-    if (!err.response) return true
-    return retryableHttpStatusCodes.has(err.response.status)
+    return RETRYABLE_HTTP_STATUS_CODES.has(err.status)
   }
   return false
-}
-
-const retryableHttpStatusCodes = new Set([
-  408, 425, 429, 500, 502, 503, 504, 522, 524,
-])
-
-export async function retryConnect<T>(
-  fn: () => Promise<T>,
-  opts: RetryOptions = {},
-): Promise<T> {
-  return retry(fn, { retryable: retryableConnect, ...opts })
-}
-
-export function retryableConnect(err: unknown) {
-  return err instanceof ConnectError && err.code === Code.Unavailable
-}
+})