@atproto/bsky 0.0.251 → 0.0.253

package/src/data-plane/server/routes/sync.ts

@@ -1,16 +0,0 @@
-import { ServiceImpl } from '@connectrpc/connect'
-import { Service } from '../../../proto/bsky_connect.js'
-import { Database } from '../db/index.js'
-
-export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
-  async getLatestRev(req) {
-    const res = await db.db
-      .selectFrom('actor_sync')
-      .where('did', '=', req.actorDid)
-      .select('repoRev')
-      .executeTakeFirst()
-    return {
-      rev: res?.repoRev ?? undefined,
-    }
-  },
-})

package/src/data-plane/server/routes/threads.ts

@@ -1,33 +0,0 @@
-import { ServiceImpl } from '@connectrpc/connect'
-import { Service } from '../../../proto/bsky_connect.js'
-import { Database } from '../db/index.js'
-import { getAncestorsAndSelfQb, getDescendentsQb } from '../util.js'
-
-export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
-  async getThread(req) {
-    const { postUri, above, below } = req
-    const [ancestors, descendents] = await Promise.all([
-      getAncestorsAndSelfQb(db.db, {
-        uri: postUri,
-        parentHeight: above,
-      })
-        .selectFrom('ancestor')
-        .selectAll()
-        .execute(),
-      getDescendentsQb(db.db, {
-        uri: postUri,
-        depth: below,
-      })
-        .selectFrom('descendent')
-        .innerJoin('post', 'post.uri', 'descendent.uri')
-        .orderBy('post.sortAt', 'desc')
-        .selectAll()
-        .execute(),
-    ])
-    const uris = [
-      ...ancestors.map((p) => p.uri),
-      ...descendents.map((p) => p.uri),
-    ]
-    return { uris }
-  },
-})

package/src/data-plane/server/subscription.ts

@@ -1,152 +0,0 @@
-import { IdResolver } from '@atproto/identity'
-import { WriteOpAction } from '@atproto/repo'
-import { Event as FirehoseEvent, Firehose, MemoryRunner } from '@atproto/sync'
-import { subLogger as log } from '../../logger.js'
-import { BackgroundQueue } from './background.js'
-import { Database } from './db/index.js'
-import { IndexingService } from './indexing/index.js'
-
-export class RepoSubscription {
-  private readonly abortController = new AbortController()
-  private current?: ReturnType<typeof createFirehose>
-  readonly background: BackgroundQueue<Database>
-  readonly indexingSvc: IndexingService
-  private readonly service: string
-  private readonly idResolver: IdResolver
-
-  constructor({
-    service,
-    db,
-    idResolver,
-  }: {
-    service: string
-    db: Database
-    idResolver: IdResolver
-  }) {
-    this.service = service
-    this.idResolver = idResolver
-    this.background = new BackgroundQueue(db, {
-      // @TODO This has historically been using the default concurrency
-      // (Infinity) but we may want to limit this in the future to avoid
-      // overloading the database with indexing tasks.
-      concurrency: Number.POSITIVE_INFINITY,
-    })
-    this.indexingSvc = new IndexingService(db, idResolver, this.background)
-  }
-
-  get destroyed() {
-    return this.abortController.signal.aborted
-  }
-
-  get running() {
-    return this.current != null
-  }
-
-  getCursor() {
-    return this.current?.runner.getCursor()
-  }
-
-  async start() {
-    this.abortController.signal.throwIfAborted()
-
-    if (!this.current) {
-      this.current = createFirehose({
-        idResolver: this.idResolver,
-        service: this.service,
-        indexingSvc: this.indexingSvc,
-      })
-      void this.current.firehose.start()
-    }
-  }
-
-  async stop() {
-    try {
-      if (this.current) {
-        const { firehose, runner } = this.current
-        this.current = undefined
-        try {
-          await firehose.destroy()
-        } finally {
-          await runner.destroy()
-        }
-      }
-    } finally {
-      await this.background.processAll()
-    }
-  }
-
-  async restart() {
-    await this.stop()
-    void this.start()
-  }
-
-  async processAll() {
-    await this.current?.runner.processAll()
-    await this.background.processAll()
-  }
-
-  async destroy() {
-    this.abortController.abort()
-    try {
-      await this.stop()
-    } finally {
-      await this.background.destroy()
-    }
-  }
-
-  async [Symbol.asyncDispose]() {
-    await this.destroy()
-  }
-}
-
-const createFirehose = (opts: {
-  idResolver: IdResolver
-  service: string
-  indexingSvc: IndexingService
-}) => {
-  const { idResolver, service, indexingSvc } = opts
-  const runner = new MemoryRunner({ startCursor: 0 })
-  const firehose = new Firehose({
-    idResolver,
-    runner,
-    service,
-    unauthenticatedHandles: true, // indexing service handles these
-    unauthenticatedCommits: true, // @TODO there seems to be a very rare issue where the authenticator thinks a block is missing in deletion ops
-    onError: (err) => log.error({ err }, 'error in subscription'),
-    handleEvent: async (evt: FirehoseEvent) => {
-      if (evt.event === 'identity') {
-        await indexingSvc.indexHandle(evt.did, evt.time, true)
-      } else if (evt.event === 'account') {
-        if (evt.active === false && evt.status === 'deleted') {
-          await indexingSvc.deleteActor(evt.did)
-        } else {
-          await indexingSvc.updateActorStatus(evt.did, evt.active, evt.status)
-        }
-      } else if (evt.event === 'sync') {
-        await Promise.all([
-          indexingSvc.setCommitLastSeen(evt.did, evt.cid, evt.rev),
-          indexingSvc.indexHandle(evt.did, evt.time),
-        ])
-      } else {
-        const indexFn =
-          evt.event === 'delete'
-            ? indexingSvc.deleteRecord(evt.uri)
-            : indexingSvc.indexRecord(
-                evt.uri,
-                evt.cid,
-                evt.record,
-                evt.event === 'create'
-                  ? WriteOpAction.Create
-                  : WriteOpAction.Update,
-                evt.time,
-              )
-        await Promise.all([
-          indexFn,
-          indexingSvc.setCommitLastSeen(evt.did, evt.commit, evt.rev),
-          indexingSvc.indexHandle(evt.did, evt.time),
-        ])
-      }
-    },
-  })
-  return { firehose, runner }
-}

package/src/data-plane/server/util.ts

@@ -1,202 +0,0 @@
-import { sql } from 'kysely'
-import { GateRecord, PostRecord, PostReplyRef } from '../../views/types.js'
-import { parseThreadGate } from '../../views/util.js'
-import { DatabaseSchema } from './db/database-schema.js'
-import { valuesList } from './db/util.js'
-
-export const getDescendentsQb = (
-  db: DatabaseSchema,
-  opts: {
-    uri: string
-    depth: number // required, protects against cycles
-  },
-) => {
-  const { uri, depth } = opts
-  const query = db.withRecursive('descendent(uri, depth)', (cte) => {
-    return cte
-      .selectFrom('post')
-      .select(['post.uri as uri', sql<number>`1`.as('depth')])
-      .where(sql`1`, '<=', depth)
-      .where('replyParent', '=', uri)
-      .unionAll(
-        cte
-          .selectFrom('post')
-          .innerJoin('descendent', 'descendent.uri', 'post.replyParent')
-          .where('descendent.depth', '<', depth)
-          .select([
-            'post.uri as uri',
-            sql<number>`descendent.depth + 1`.as('depth'),
-          ]),
-      )
-  })
-  return query
-}
-
-export const getAncestorsAndSelfQb = (
-  db: DatabaseSchema,
-  opts: {
-    uri: string
-    parentHeight: number // required, protects against cycles
-  },
-) => {
-  const { uri, parentHeight } = opts
-  const query = db.withRecursive(
-    'ancestor(uri, ancestorUri, height)',
-    (cte) => {
-      return cte
-        .selectFrom('post')
-        .select([
-          'post.uri as uri',
-          'post.replyParent as ancestorUri',
-          sql<number>`0`.as('height'),
-        ])
-        .where('uri', '=', uri)
-        .unionAll(
-          cte
-            .selectFrom('post')
-            .innerJoin('ancestor', 'ancestor.ancestorUri', 'post.uri')
-            .where('ancestor.height', '<', parentHeight)
-            .select([
-              'post.uri as uri',
-              'post.replyParent as ancestorUri',
-              sql<number>`ancestor.height + 1`.as('height'),
-            ]),
-        )
-    },
-  )
-  return query
-}
-
-export const invalidReplyRoot = (
-  reply: PostReplyRef,
-  parent: {
-    record: PostRecord
-    invalidReplyRoot: boolean | null
-  },
-) => {
-  const replyRoot = reply.root.uri
-  const replyParent = reply.parent.uri
-  // if parent is not a valid reply, transitively this is not a valid one either
-  if (parent.invalidReplyRoot) {
-    return true
-  }
-  // replying to root post: ensure the root looks correct
-  if (replyParent === replyRoot) {
-    return !!parent.record.reply
-  }
-  // replying to a reply: ensure the parent is a reply for the same root post
-  return parent.record.reply?.root.uri !== replyRoot
-}
-export const violatesThreadGate = async (
-  db: DatabaseSchema,
-  replierDid: string,
-  ownerDid: string,
-  rootPost: PostRecord | null,
-  gate: GateRecord | null,
-) => {
-  const {
-    canReply,
-    allowFollower,
-    allowFollowing,
-    allowListUris = [],
-  } = parseThreadGate(replierDid, ownerDid, rootPost, gate)
-  if (canReply) {
-    return false
-  }
-  if (!allowFollower && !allowFollowing && !allowListUris?.length) {
-    return true
-  }
-  const { ref } = db.dynamic
-  const nullResult = sql<null>`${null}`
-  const check = await db
-    .selectFrom(valuesList([replierDid]).as(sql`subject (did)`))
-    .select([
-      allowFollower
-        ? db
-            .selectFrom('follow')
-            .where('subjectDid', '=', ownerDid)
-            .whereRef('creator', '=', ref('subject.did'))
-            .select('subjectDid')
-            .as('isFollower')
-        : nullResult.as('isFollower'),
-      allowFollowing
-        ? db
-            .selectFrom('follow')
-            .where('creator', '=', ownerDid)
-            .whereRef('subjectDid', '=', ref('subject.did'))
-            .select('creator')
-            .as('isFollowed')
-        : nullResult.as('isFollowed'),
-      allowListUris.length
-        ? db
-            .selectFrom('list_item')
-            .where('list_item.listUri', 'in', allowListUris)
-            .whereRef('list_item.subjectDid', '=', ref('subject.did'))
-            .limit(1)
-            .select('listUri')
-            .as('isInList')
-        : nullResult.as('isInList'),
-    ])
-    .executeTakeFirst()
-
-  if (allowFollowing && check?.isFollowed) {
-    return false
-  } else if (allowFollower && check?.isFollower) {
-    return false
-  } else if (allowListUris.length && check?.isInList) {
-    return false
-  }
-
-  return true
-}
-
-// @NOTE: This type is not complete with all supported options.
-// Only the ones that we needed to apply custom logic on are currently present.
-export type PostSearchQuery = {
-  q: string
-  author: string | undefined
-}
-
-export const parsePostSearchQuery = (
-  qParam: string,
-  params?: {
-    author?: string
-  },
-): PostSearchQuery => {
-  // Accept individual params, but give preference to options embedded in `q`.
-  let author = params?.author
-
-  const parts: string[] = []
-  let curr = ''
-  let quoted = false
-  for (const c of qParam) {
-    if (c === ' ' && !quoted) {
-      if (curr.trim()) parts.push(curr)
-      curr = ''
-      continue
-    }
-
-    if (c === '"') {
-      quoted = !quoted
-    }
-    curr += c
-  }
-  if (curr.trim()) parts.push(curr)
-
-  const qParts: string[] = []
-  for (const p of parts) {
-    const tokens = p.split(':')
-    if (tokens[0] === 'did') {
-      author = p
-    } else if (tokens[0] === 'author' || tokens[0] === 'from') {
-      author = tokens[1]
-    } else {
-      qParts.push(p)
-    }
-  }
-
-  return {
-    q: qParts.join(' '),
-    author,
-  }
-}

package/src/error.ts

@@ -1,12 +0,0 @@
-import { ErrorRequestHandler } from 'express'
-import { XRPCError } from '@atproto/xrpc-server'
-import { httpLogger as log } from './logger.js'
-
-export const handler: ErrorRequestHandler = (err, _req, res, next) => {
-  log.error({ err }, 'unexpected internal server error')
-  if (res.headersSent) {
-    return next(err)
-  }
-  const serverError = XRPCError.fromError(err)
-  res.status(serverError.type).json(serverError.payload)
-}

package/src/etcd.ts

@@ -1,90 +0,0 @@
-import { once } from 'node:events'
-import { Etcd3, Watcher } from 'etcd3'
-
-/**
- * A reactive map based on the keys and values stored within etcd under a given prefix.
- */
-export class EtcdMap {
-  inner = new Map<string, VersionedValue>()
-  watcher: Watcher
-  connecting: Promise<void> | undefined
-  handlers: ((self: EtcdMap) => void)[] = []
-
-  constructor(
-    private etcd: Etcd3,
-    private prefix = '',
-  ) {
-    this.watcher = etcd.watch().prefix(prefix).watcher()
-    this.connecting = connectWatcher(this.watcher)
-  }
-
-  async connect() {
-    this.watcher.on('put', (kv) => {
-      const key = kv.key.toString()
-      const value = kv.value.toString()
-      const rev = revToInt(kv.mod_revision)
-      this.apply(key, { value, rev })
-    })
-    this.watcher.on('delete', (kv) => {
-      const key = kv.key.toString()
-      const value = null
-      const rev = revToInt(kv.mod_revision)
-      this.apply(key, { value, rev })
-    })
-    await this.connecting?.finally(() => {
-      this.connecting = undefined
-    })
-    const { kvs } = await this.etcd.getAll().prefix(this.prefix).exec()
-    for (const kv of kvs) {
-      const key = kv.key.toString()
-      const value = kv.value.toString()
-      const rev = revToInt(kv.mod_revision)
-      this.apply(key, { value, rev })
-    }
-  }
-
-  get(key: string) {
-    return this.inner.get(key)?.value ?? null
-  }
-
-  *values() {
-    for (const { value } of this.inner.values()) {
-      if (value !== null) {
-        yield value
-      }
-    }
-  }
-
-  onUpdate(handler: (self: EtcdMap) => void) {
-    this.handlers.push(handler)
-  }
-
-  private update() {
-    for (const handler of this.handlers) {
-      handler(this)
-    }
-  }
-
-  private apply(key, vv: VersionedValue) {
-    const curr = this.inner.get(key)
-    if (curr && curr.rev > vv.rev) return
-    this.inner.set(key, vv)
-    this.update()
-  }
-}
-
-function revToInt(rev: string) {
-  return parseInt(rev, 10)
-}
-
-async function connectWatcher(watcher: Watcher) {
-  await Promise.race([
-    once(watcher, 'connected'),
-    once(watcher, 'error').then((err) => Promise.reject(err)),
-  ])
-}
-
-type VersionedValue = {
-  rev: number
-  value: string | null
-}

package/src/feature-gates/README.md

@@ -1,83 +0,0 @@
-# Feature Gates
-
-A thin wrapper around [GrowthBook](https://www.growthbook.io/) for feature flag
-evaluation and experiment tracking.
-
-## Usage
-
-Feature gates are accessible on `HydrationCtx` as `features`. A default "scope"
-is defined in `Hydrator.createContext`, which is called by every request
-handler. The default scope supplies anonymous `deviceId` and `sessionId`
-identifiers for targeting unauthenticated users.
-
-Feature gates can be checked via the following methods.
-
-```typescript
-// check a single gate
-const enabled = ctx.features.checkGate(
-  ctx.features.Gate.ThreadsReplyRankingExplorationEnable,
-)
-
-// check multiple gates
-const gates = ctx.features.checkGates([
-  ctx.features.Gate.ThreadsReplyRankingExplorationEnable,
-  ctx.features.Gate.SomeOtherGate,
-])
-const enabled = gates.get(
-  ctx.features.Gate.ThreadsReplyRankingExplorationEnable,
-)
-```
-
-### User Context
-
-To accurately gate features by user, we need additional context about that user.
-For most use cases, we get this data from within the request handlers.
-
-Here, the `features` client will be scoped to the current user, allowing for
-accurate gate checks throughout the request lifecycle.
-
-```typescript
-const features: ScopedFeatureGatesClient = ctx.featureGatesClient.scope(
-  ctx.featureGatesClient.parseUserContextFromHandler({
-    viewer,
-    req,
-  }),
-)
-
-const hydrateCtx = await ctx.hydrator.createContext({
-  labelers,
-  viewer,
-  features,
-})
-```
-
-> [!NOTE]
-> Although `ctx.featureGatesClient` also has a `checkGates` method, the
-> intention is to use the `ScopedFeatureGatesClient` returned by `scope()` for
-> gate checks within the application.
-
-You can also pass the user context directly to `checkGate` or `checkGates`. This
-overrides any default scope set on the client, allowing for flexibility in cases
-where user context is only available at the point of gate evaluation.
-
-```typescript
-const enabled = ctx.featureGatesClient.checkGate(
-  ctx.featureGatesClient.Gate.ImageFeatureEnabled,
-  { did: imageAuthor.did },
-)
-```
-
-### Adding Gates
-
-See `gates.ts` and add them to the enum. You can optionally prevent metrics from
-being emitted for a gate by adding the gate to the `IGNORE_METRICS_FOR_GATES`
-set.
-
-## Metrics
-
-Check out the Growthbook docs for more info here. Basically, every feature eval
-fires the `onFeatureUsage` callback. Any feature that is part of an experiment
-_will also fire the `trackingCallback` callback._
-
-For some use cases, this can be noisy and/or performance intensive, so make use
-of `IGNORE_METRICS_FOR_GATES` to silence metrics for specific gates.

package/src/feature-gates/gates.ts

@@ -1,25 +0,0 @@
-/**
- * Enum of all gates in the system. This should be the single source of truth
- * for all gates, and should be used in all places where gates are checked or
- * defined.
- */
-export enum Gate {
-  SuggestedUsersDiscoverEnable = 'suggested_users:discover_agent:enable',
-  SuggestedUsersSocialProofEnable = 'suggested_users:social_proof:enable',
-  ThreadsReplyRankingExplorationEnable = 'threads:reply_ranking_exploration:enable',
-  SearchFilteringExplorationEnable = 'search:filtering_exploration:enable',
-  SuggestedUsersForExploreEnable = 'suggested_users:for_explore:enable',
-  SuggestedUsersForDiscoverEnable = 'suggested_users:for_discover:enable',
-  SuggestedUsersForSeeMoreEnable = 'suggested_users:for_see_more:enable',
-  SearchV2Enable = 'search:v2:enable',
-
-  // temp
-  AATest = 'aa-test-appview',
-}
-
-/**
- * Set of gates that should be ignored when tracking gate evaluations for
- * analytics purposes. This is useful for gates that are not user-facing or are
- * overly noisy.
- */
-export const IGNORE_METRICS_FOR_GATES: Set<Gate> = new Set([])