@atproto/bsky 0.0.16 → 0.0.18

package/src/services/feed/views.ts

@@ -1,4 +1,5 @@
 import { mapDefined } from '@atproto/common'
+import { AtUri } from '@atproto/syntax'
 import { Database } from '../../db'
 import {
   FeedViewPost,
@@ -37,26 +38,35 @@ import {
 } from './types'
 import { Labels, getSelfLabels } from '../label'
 import { ImageUriBuilder } from '../../image/uri'
-import { LabelCache } from '../../label-cache'
 import { ActorInfoMap, ActorService } from '../actor'
 import { ListInfoMap, GraphService } from '../graph'
-import { AtUri } from '@atproto/syntax'
+import { FromDb } from '../types'
 import { parseThreadGate } from './util'
 
 export class FeedViews {
+  services: {
+    actor: ActorService
+    graph: GraphService
+  }
+
   constructor(
     public db: Database,
     public imgUriBuilder: ImageUriBuilder,
-    public labelCache: LabelCache,
-  ) {}
-
-  static creator(imgUriBuilder: ImageUriBuilder, labelCache: LabelCache) {
-    return (db: Database) => new FeedViews(db, imgUriBuilder, labelCache)
+    private actor: FromDb<ActorService>,
+    private graph: FromDb<GraphService>,
+  ) {
+    this.services = {
+      actor: actor(this.db),
+      graph: graph(this.db),
+    }
   }
 
-  services = {
-    actor: ActorService.creator(this.imgUriBuilder, this.labelCache)(this.db),
-    graph: GraphService.creator(this.imgUriBuilder)(this.db),
+  static creator(
+    imgUriBuilder: ImageUriBuilder,
+    actor: FromDb<ActorService>,
+    graph: FromDb<GraphService>,
+  ) {
+    return (db: Database) => new FeedViews(db, imgUriBuilder, actor, graph)
   }
 
   formatFeedGeneratorView(

package/src/services/index.ts

@@ -1,25 +1,29 @@
-import { Database, PrimaryDatabase } from '../db'
 import { ImageUriBuilder } from '../image/uri'
 import { ActorService } from './actor'
 import { FeedService } from './feed'
 import { GraphService } from './graph'
 import { ModerationService } from './moderation'
-import { LabelService } from './label'
+import { LabelCacheOpts, LabelService } from './label'
 import { ImageInvalidator } from '../image/invalidator'
-import { LabelCache } from '../label-cache'
+import { FromDb, FromDbPrimary } from './types'
 
 export function createServices(resources: {
   imgUriBuilder: ImageUriBuilder
   imgInvalidator: ImageInvalidator
-  labelCache: LabelCache
+  labelCacheOpts: LabelCacheOpts
 }): Services {
-  const { imgUriBuilder, imgInvalidator, labelCache } = resources
+  const { imgUriBuilder, imgInvalidator, labelCacheOpts } = resources
+  const label = LabelService.creator(labelCacheOpts)
+  const graph = GraphService.creator(imgUriBuilder)
+  const actor = ActorService.creator(imgUriBuilder, graph, label)
+  const moderation = ModerationService.creator(imgUriBuilder, imgInvalidator)
+  const feed = FeedService.creator(imgUriBuilder, actor, label, graph)
   return {
-    actor: ActorService.creator(imgUriBuilder, labelCache),
-    feed: FeedService.creator(imgUriBuilder, labelCache),
-    graph: GraphService.creator(imgUriBuilder),
-    moderation: ModerationService.creator(imgUriBuilder, imgInvalidator),
-    label: LabelService.creator(labelCache),
+    actor,
+    feed,
+    moderation,
+    graph,
+    label,
   }
 }
 
@@ -30,7 +34,3 @@ export type Services = {
   moderation: FromDbPrimary<ModerationService>
   label: FromDb<LabelService>
 }
-
-type FromDb<T> = (db: Database) => T
-
-type FromDbPrimary<T> = (db: PrimaryDatabase) => T

package/src/services/indexing/index.ts

@@ -7,6 +7,7 @@ import {
   verifyRepo,
   Commit,
   VerifiedRepo,
+  getAndParseRecord,
 } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
 import { IdResolver, getPds } from '@atproto/identity'
@@ -201,10 +202,11 @@ export class IndexingService {
           if (op.op === 'delete') {
             await this.deleteRecord(uri)
           } else {
+            const parsed = await getAndParseRecord(blocks, cid)
             await this.indexRecord(
               uri,
               cid,
-              op.value,
+              parsed.record,
               op.op === 'create' ? WriteOpAction.Create : WriteOpAction.Update,
               now,
             )
@@ -389,19 +391,15 @@ type UriAndCid = {
   cid: CID
 }
 
-type RecordDescript = UriAndCid & {
-  value: unknown
-}
-
 type IndexOp =
   | ({
       op: 'create' | 'update'
-    } & RecordDescript)
+    } & UriAndCid)
   | ({ op: 'delete' } & UriAndCid)
 
 const findDiffFromCheckout = (
   curr: Record<string, UriAndCid>,
-  checkout: Record<string, RecordDescript>,
+  checkout: Record<string, UriAndCid>,
 ): IndexOp[] => {
   const ops: IndexOp[] = []
   for (const uri of Object.keys(checkout)) {
@@ -428,14 +426,13 @@ const findDiffFromCheckout = (
 const formatCheckout = (
   did: string,
   verifiedRepo: VerifiedRepo,
-): Record<string, RecordDescript> => {
-  const records: Record<string, RecordDescript> = {}
+): Record<string, UriAndCid> => {
+  const records: Record<string, UriAndCid> = {}
   for (const create of verifiedRepo.creates) {
     const uri = AtUri.make(did, create.collection, create.rkey)
     records[uri.toString()] = {
       uri,
       cid: create.cid,
-      value: create.record,
     }
   }
   return records

package/src/services/indexing/plugins/post.ts

@@ -112,6 +112,7 @@ const insertFn = async (
       obj.reply,
     )
     if (invalidReplyRoot || violatesThreadGate) {
+      Object.assign(insertedPost, { invalidReplyRoot, violatesThreadGate })
       await db
         .updateTable('post')
         .where('uri', '=', post.uri)
@@ -241,6 +242,13 @@ const notifsForInsert = (obj: IndexedPost) => {
     }
   }
 
+  if (obj.post.violatesThreadGate) {
+    // don't generate reply notifications when post violates threadgate
+    return notifs
+  }
+
+  // reply notifications
+
   for (const ancestor of obj.ancestors ?? []) {
     if (ancestor.uri === obj.post.uri) continue // no need to notify for own post
     if (ancestor.height < REPLY_NOTIF_DEPTH) {
@@ -353,6 +361,11 @@ const updateAggregates = async (db: DatabaseSchema, postIdx: IndexedPost) => {
           replyCount: db
             .selectFrom('post')
             .where('post.replyParent', '=', postIdx.post.replyParent)
+            .where((qb) =>
+              qb
+                .where('post.violatesThreadGate', 'is', null)
+                .orWhere('post.violatesThreadGate', '=', false),
+            )
             .select(countAll.as('count')),
         })
         .onConflict((oc) =>

package/src/services/label/index.ts

@@ -3,15 +3,36 @@ import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { Database } from '../../db'
 import { Label, isSelfLabels } from '../../lexicon/types/com/atproto/label/defs'
 import { ids } from '../../lexicon/lexicons'
-import { LabelCache } from '../../label-cache'
+import { ReadThroughCache } from '../../cache/read-through'
+import { Redis } from '../../redis'
 
 export type Labels = Record<string, Label[]>
 
+export type LabelCacheOpts = {
+  redis: Redis
+  staleTTL: number
+  maxTTL: number
+}
+
 export class LabelService {
-  constructor(public db: Database, public cache: LabelCache | null) {}
+  public cache: ReadThroughCache<Label[]> | null
 
-  static creator(cache: LabelCache | null) {
-    return (db: Database) => new LabelService(db, cache)
+  constructor(public db: Database, cacheOpts: LabelCacheOpts | null) {
+    if (cacheOpts) {
+      this.cache = new ReadThroughCache(cacheOpts.redis, {
+        ...cacheOpts,
+        fetchMethod: async (subject: string) => {
+          const res = await fetchLabelsForSubjects(db, [subject])
+          return res[subject] ?? []
+        },
+        fetchManyMethod: (subjects: string[]) =>
+          fetchLabelsForSubjects(db, subjects),
+      })
+    }
+  }
+
+  static creator(cacheOpts: LabelCacheOpts | null) {
+    return (db: Database) => new LabelService(db, cacheOpts)
   }
 
   async formatAndCreate(
@@ -72,24 +93,19 @@ export class LabelService {
     },
   ): Promise<Labels> {
     if (subjects.length < 1) return {}
-    const res =
-      this.cache === null || opts?.skipCache
-        ? await this.db.db
-            .selectFrom('label')
-            .where('label.uri', 'in', subjects)
-            .if(!opts?.includeNeg, (qb) => qb.where('neg', '=', false))
-            .selectAll()
-            .execute()
-        : this.cache.forSubjects(subjects, opts?.includeNeg)
-    return res.reduce((acc, cur) => {
-      acc[cur.uri] ??= []
-      acc[cur.uri].push({
-        ...cur,
-        cid: cur.cid === '' ? undefined : cur.cid,
-        neg: cur.neg,
-      })
-      return acc
-    }, {} as Labels)
+    const res = this.cache
+      ? await this.cache.getMany(subjects, { revalidate: opts?.skipCache })
+      : await fetchLabelsForSubjects(this.db, subjects)
+
+    if (opts?.includeNeg) {
+      return res
+    }
+
+    const noNegs: Labels = {}
+    for (const [key, val] of Object.entries(res)) {
+      noNegs[key] = val.filter((label) => !label.neg)
+    }
+    return noNegs
   }
 
   // gets labels for any record. when did is present, combine labels for both did & profile record.
@@ -171,3 +187,31 @@ export function getSelfLabels(details: {
     return { src, uri, cid, val, cts, neg: false }
   })
 }
+
+const fetchLabelsForSubjects = async (
+  db: Database,
+  subjects: string[],
+): Promise<Record<string, Label[]>> => {
+  if (subjects.length === 0) {
+    return {}
+  }
+  const res = await db.db
+    .selectFrom('label')
+    .where('label.uri', 'in', subjects)
+    .selectAll()
+    .execute()
+  const labelMap = res.reduce((acc, cur) => {
+    acc[cur.uri] ??= []
+    acc[cur.uri].push({
+      ...cur,
+      cid: cur.cid === '' ? undefined : cur.cid,
+      neg: cur.neg,
+    })
+    return acc
+  }, {} as Record<string, Label[]>)
+  // ensure we cache negatives
+  for (const subject of subjects) {
+    labelMap[subject] ??= []
+  }
+  return labelMap
+}

package/src/services/moderation/index.ts

@@ -639,7 +639,7 @@ export class ModerationService {
     const { did, recordPath } = getStatusIdentifierFromSubject(
       'did' in subject ? subject.did : subject.uri,
     )
-    let builder = this.db.db
+    const builder = this.db.db
       .selectFrom('moderation_subject_status')
       .where('did', '=', did)
       .where('recordPath', '=', recordPath || '')

package/src/services/moderation/status.ts

@@ -2,10 +2,7 @@
 
 import { AtUri } from '@atproto/syntax'
 import { PrimaryDatabase } from '../../db'
-import {
-  ModerationEvent,
-  ModerationSubjectStatus,
-} from '../../db/tables/moderation'
+import { ModerationSubjectStatus } from '../../db/tables/moderation'
 import {
   REVIEWOPEN,
   REVIEWCLOSED,

package/src/services/types.ts

@@ -0,0 +1,4 @@
+import { Database, PrimaryDatabase } from '../db'
+
+export type FromDb<T> = (db: Database) => T
+export type FromDbPrimary<T> = (db: PrimaryDatabase) => T

package/src/services/util/notification.ts

@@ -0,0 +1,70 @@
+import { sql } from 'kysely'
+import { countAll } from '../../db/util'
+import { PrimaryDatabase } from '../../db'
+
+// i.e. 30 days before the last time the user checked their notifs
+export const BEFORE_LAST_SEEN_DAYS = 30
+// i.e. 180 days before the latest unread notification
+export const BEFORE_LATEST_UNREAD_DAYS = 180
+// don't consider culling unreads until they hit this threshold, and then enforce beforeLatestUnreadThresholdDays
+export const UNREAD_KEPT_COUNT = 500
+
+export const tidyNotifications = async (db: PrimaryDatabase, did: string) => {
+  const stats = await db.db
+    .selectFrom('notification')
+    .select([
+      sql<0 | 1>`("sortAt" < "lastSeenNotifs")`.as('read'),
+      countAll.as('count'),
+      sql<string>`min("sortAt")`.as('earliestAt'),
+      sql<string>`max("sortAt")`.as('latestAt'),
+      sql<string>`max("lastSeenNotifs")`.as('lastSeenAt'),
+    ])
+    .leftJoin('actor_state', 'actor_state.did', 'notification.did')
+    .where('notification.did', '=', did)
+    .groupBy(sql`1`) // group by read (i.e. 1st column)
+    .execute()
+  const readStats = stats.find((stat) => stat.read)
+  const unreadStats = stats.find((stat) => !stat.read)
+  let readCutoffAt: Date | undefined
+  let unreadCutoffAt: Date | undefined
+  if (readStats) {
+    readCutoffAt = addDays(
+      new Date(readStats.lastSeenAt),
+      -BEFORE_LAST_SEEN_DAYS,
+    )
+  }
+  if (unreadStats && unreadStats.count > UNREAD_KEPT_COUNT) {
+    unreadCutoffAt = addDays(
+      new Date(unreadStats.latestAt),
+      -BEFORE_LATEST_UNREAD_DAYS,
+    )
+  }
+  // take most recent of read/unread cutoffs
+  const cutoffAt = greatest(readCutoffAt, unreadCutoffAt)
+  if (cutoffAt) {
+    // skip delete if it wont catch any notifications
+    const earliestAt = least(readStats?.earliestAt, unreadStats?.earliestAt)
+    if (earliestAt && earliestAt < cutoffAt.toISOString()) {
+      await db.db
+        .deleteFrom('notification')
+        .where('did', '=', did)
+        .where('sortAt', '<', cutoffAt.toISOString())
+        .execute()
+    }
+  }
+}
+
+const addDays = (date: Date, days: number) => {
+  date.setDate(date.getDate() + days)
+  return date
+}
+
+const least = <T extends Ordered>(a: T | undefined, b: T | undefined) => {
+  return a !== undefined && (b === undefined || a < b) ? a : b
+}
+
+const greatest = <T extends Ordered>(a: T | undefined, b: T | undefined) => {
+  return a !== undefined && (b === undefined || a > b) ? a : b
+}
+
+type Ordered = string | number | Date

package/src/util/retry.ts

@@ -1,26 +1,6 @@
 import { AxiosError } from 'axios'
-import { wait } from '@atproto/common'
 import { XRPCError, ResponseType } from '@atproto/xrpc'
-
-export async function retry<T>(
-  fn: () => Promise<T>,
-  opts: RetryOptions = {},
-): Promise<T> {
-  const { max = 3, retryable = () => true } = opts
-  let retries = 0
-  let doneError: unknown
-  while (!doneError) {
-    try {
-      if (retries) await backoff(retries)
-      return await fn()
-    } catch (err) {
-      const willRetry = retries < max && retryable(err)
-      if (!willRetry) doneError = err
-      retries += 1
-    }
-  }
-  throw doneError
-}
+import { RetryOptions, retry } from '@atproto/common'
 
 export async function retryHttp<T>(
   fn: () => Promise<T>,
@@ -44,26 +24,3 @@ export function retryableHttp(err: unknown) {
 const retryableHttpStatusCodes = new Set([
   408, 425, 429, 500, 502, 503, 504, 522, 524,
 ])
-
-type RetryOptions = {
-  max?: number
-  retryable?: (err: unknown) => boolean
-}
-
-// Waits exponential backoff with max and jitter: ~50, ~100, ~200, ~400, ~800, ~1000, ~1000, ...
-async function backoff(n: number, multiplier = 50, max = 1000) {
-  const exponentialMs = Math.pow(2, n) * multiplier
-  const ms = Math.min(exponentialMs, max)
-  await wait(jitter(ms))
-}
-
-// Adds randomness +/-15% of value
-function jitter(value: number) {
-  const delta = value * 0.15
-  return value + randomRange(-delta, delta)
-}
-
-function randomRange(from: number, to: number) {
-  const rand = Math.random() * (to - from)
-  return rand + from
-}

package/tests/admin/get-repo.test.ts

@@ -97,9 +97,11 @@ describe('admin get repo view', () => {
     expect(beforeEmailVerification.emailConfirmedAt).toBeUndefined()
     const timestampBeforeVerification = Date.now()
     const bobsAccount = sc.accounts[sc.dids.bob]
-    const verificationToken = await network.pds.ctx.services
-      .account(network.pds.ctx.db)
-      .createEmailToken(sc.dids.bob, 'confirm_email')
+    const verificationToken =
+      await network.pds.ctx.accountManager.createEmailToken(
+        sc.dids.bob,
+        'confirm_email',
+      )
     await agent.api.com.atproto.server.confirmEmail(
       { email: bobsAccount.email, token: verificationToken },
       {

package/tests/admin/moderation.test.ts

@@ -387,7 +387,7 @@ describe('moderation', () => {
         createLabelVals: ['nsfw'],
         negateLabelVals: [],
       })
-      const { data: takedownAction } = await emitModEvent({
+      await emitModEvent({
         $type: 'com.atproto.admin.defs#modEventTakedown',
       })
 
@@ -779,7 +779,7 @@ describe('moderation', () => {
         negateLabelVals: ModEventLabel['negateLabelVals']
       },
     ) {
-      const { createLabelVals, negateLabelVals, ...rest } = opts
+      const { createLabelVals, negateLabelVals } = opts
       const result = await agent.api.com.atproto.admin.emitModerationEvent(
         {
           event: {

package/tests/admin/repo-search.test.ts

@@ -17,6 +17,7 @@ describe('admin repo search view', () => {
     sc = network.getSeedClient()
     await usersBulkSeed(sc)
     headers = network.pds.adminAuthHeaders()
+    await network.processAll()
   })
 
   afterAll(async () => {

package/tests/algos/hot-classic.test.ts

@@ -31,7 +31,6 @@ describe('algo hot-classic', () => {
     alice = sc.dids.alice
     bob = sc.dids.bob
     await network.processAll()
-    await network.bsky.processAll()
   })
 
   afterAll(async () => {
@@ -59,7 +58,7 @@ describe('algo hot-classic', () => {
       await sc.like(sc.dids[name], two.ref)
       await sc.like(sc.dids[name], three.ref)
     }
-    await network.bsky.processAll()
+    await network.processAll()
 
     const res = await agent.api.app.bsky.feed.getFeed(
       { feed: feedUri },

package/tests/auth.test.ts

@@ -36,7 +36,7 @@ describe('auth', () => {
         { headers: { authorization: `Bearer ${jwt}` } },
       )
     }
-    const origSigningKey = network.pds.ctx.repoSigningKey
+    const origSigningKey = await network.pds.ctx.actorStore.keypair(issuer)
     const newSigningKey = await Secp256k1Keypair.create({ exportable: true })
     // confirm original signing key works
     await expect(attemptWithKey(origSigningKey)).resolves.toBeDefined()

package/tests/auto-moderator/labeler.test.ts

@@ -40,26 +40,25 @@ describe('labeler', () => {
     await usersSeed(sc)
     await network.processAll()
     alice = sc.dids.alice
-    const repoSvc = pdsCtx.services.repo(pdsCtx.db)
-    const storeBlob = async (bytes: Uint8Array) => {
-      const blobRef = await repoSvc.blobs.addUntetheredBlob(
-        alice,
-        'image/jpeg',
-        Readable.from([bytes], { objectMode: false }),
-      )
-      const preparedBlobRef = {
-        cid: blobRef.ref,
-        mimeType: 'image/jpeg',
-        constraints: {},
-      }
-      await repoSvc.blobs.verifyBlobAndMakePermanent(alice, preparedBlobRef)
-      await repoSvc.blobs.associateBlob(
-        preparedBlobRef,
-        postUri(),
-        TID.nextStr(),
-        alice,
-      )
-      return blobRef
+    const storeBlob = (bytes: Uint8Array) => {
+      return pdsCtx.actorStore.transact(alice, async (store) => {
+        const blobRef = await store.repo.blob.addUntetheredBlob(
+          'image/jpeg',
+          Readable.from([bytes], { objectMode: false }),
+        )
+        const preparedBlobRef = {
+          cid: blobRef.ref,
+          mimeType: 'image/jpeg',
+          constraints: {},
+        }
+        await store.repo.blob.verifyBlobAndMakePermanent(preparedBlobRef)
+        await store.repo.blob.associateBlob(
+          preparedBlobRef,
+          postUri(),
+          TID.nextStr(),
+        )
+        return blobRef
+      })
     }
     const bytes1 = new Uint8Array([1, 2, 3, 4])
     const bytes2 = new Uint8Array([5, 6, 7, 8])

package/tests/auto-moderator/takedowns.test.ts

@@ -106,11 +106,15 @@ describe('takedowner', () => {
       .executeTakeFirst()
     expect(record?.takedownId).toBeGreaterThan(0)
 
-    const recordPds = await network.pds.ctx.db.db
-      .selectFrom('record')
-      .where('uri', '=', post.ref.uriStr)
-      .select('takedownRef')
-      .executeTakeFirst()
+    const recordPds = await network.pds.ctx.actorStore.read(
+      post.ref.uri.hostname,
+      (store) =>
+        store.db.db
+          .selectFrom('record')
+          .where('uri', '=', post.ref.uriStr)
+          .select('takedownRef')
+          .executeTakeFirst(),
+    )
     expect(recordPds?.takedownRef).toEqual(takedownEvent.id.toString())
 
     expect(testInvalidator.invalidated.length).toBe(1)
@@ -162,11 +166,13 @@ describe('takedowner', () => {
       .executeTakeFirst()
     expect(record?.takedownId).toBeGreaterThan(0)
 
-    const recordPds = await network.pds.ctx.db.db
-      .selectFrom('record')
-      .where('uri', '=', res.data.uri)
-      .select('takedownRef')
-      .executeTakeFirst()
+    const recordPds = await network.pds.ctx.actorStore.read(alice, (store) =>
+      store.db.db
+        .selectFrom('record')
+        .where('uri', '=', res.data.uri)
+        .select('takedownRef')
+        .executeTakeFirst(),
+    )
     expect(recordPds?.takedownRef).toEqual(takedownEvent.id.toString())
 
     expect(testInvalidator.invalidated.length).toBe(2)

package/tests/blob-resolver.test.ts

@@ -77,8 +77,10 @@ describe('blob resolver', () => {
   })
 
   it('fails on blob with bad signature check.', async () => {
-    await network.pds.ctx.blobstore.delete(fileCid)
-    await network.pds.ctx.blobstore.putPermanent(fileCid, randomBytes(100))
+    await network.pds.ctx.blobstore(fileDid).delete(fileCid)
+    await network.pds.ctx
+      .blobstore(fileDid)
+      .putPermanent(fileCid, randomBytes(100))
     const tryGetBlob = client.get(`/blob/${fileDid}/${fileCid.toString()}`)
     await expect(tryGetBlob).rejects.toThrow(
       'maxContentLength size of -1 exceeded',