@atproto/bsky 0.0.15 → 0.0.17

package/src/services/index.ts

@@ -1,25 +1,29 @@
-import { Database, PrimaryDatabase } from '../db'
 import { ImageUriBuilder } from '../image/uri'
 import { ActorService } from './actor'
 import { FeedService } from './feed'
 import { GraphService } from './graph'
 import { ModerationService } from './moderation'
-import { LabelService } from './label'
+import { LabelCacheOpts, LabelService } from './label'
 import { ImageInvalidator } from '../image/invalidator'
-import { LabelCache } from '../label-cache'
+import { FromDb, FromDbPrimary } from './types'
 
 export function createServices(resources: {
   imgUriBuilder: ImageUriBuilder
   imgInvalidator: ImageInvalidator
-  labelCache: LabelCache
+  labelCacheOpts: LabelCacheOpts
 }): Services {
-  const { imgUriBuilder, imgInvalidator, labelCache } = resources
+  const { imgUriBuilder, imgInvalidator, labelCacheOpts } = resources
+  const label = LabelService.creator(labelCacheOpts)
+  const graph = GraphService.creator(imgUriBuilder)
+  const actor = ActorService.creator(imgUriBuilder, graph, label)
+  const moderation = ModerationService.creator(imgUriBuilder, imgInvalidator)
+  const feed = FeedService.creator(imgUriBuilder, actor, label, graph)
   return {
-    actor: ActorService.creator(imgUriBuilder, labelCache),
-    feed: FeedService.creator(imgUriBuilder, labelCache),
-    graph: GraphService.creator(imgUriBuilder),
-    moderation: ModerationService.creator(imgUriBuilder, imgInvalidator),
-    label: LabelService.creator(labelCache),
+    actor,
+    feed,
+    moderation,
+    graph,
+    label,
   }
 }
 
@@ -30,7 +34,3 @@ export type Services = {
   moderation: FromDbPrimary<ModerationService>
   label: FromDb<LabelService>
 }
-
-type FromDb<T> = (db: Database) => T
-
-type FromDbPrimary<T> = (db: PrimaryDatabase) => T

package/src/services/indexing/index.ts

@@ -7,6 +7,7 @@ import {
   verifyRepo,
   Commit,
   VerifiedRepo,
+  getAndParseRecord,
 } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
 import { IdResolver, getPds } from '@atproto/identity'
@@ -201,10 +202,11 @@ export class IndexingService {
           if (op.op === 'delete') {
             await this.deleteRecord(uri)
           } else {
+            const parsed = await getAndParseRecord(blocks, cid)
             await this.indexRecord(
               uri,
               cid,
-              op.value,
+              parsed.record,
               op.op === 'create' ? WriteOpAction.Create : WriteOpAction.Update,
               now,
             )
@@ -389,19 +391,15 @@ type UriAndCid = {
   cid: CID
 }
 
-type RecordDescript = UriAndCid & {
-  value: unknown
-}
-
 type IndexOp =
   | ({
       op: 'create' | 'update'
-    } & RecordDescript)
+    } & UriAndCid)
   | ({ op: 'delete' } & UriAndCid)
 
 const findDiffFromCheckout = (
   curr: Record<string, UriAndCid>,
-  checkout: Record<string, RecordDescript>,
+  checkout: Record<string, UriAndCid>,
 ): IndexOp[] => {
   const ops: IndexOp[] = []
   for (const uri of Object.keys(checkout)) {
@@ -428,14 +426,13 @@ const findDiffFromCheckout = (
 const formatCheckout = (
   did: string,
   verifiedRepo: VerifiedRepo,
-): Record<string, RecordDescript> => {
-  const records: Record<string, RecordDescript> = {}
+): Record<string, UriAndCid> => {
+  const records: Record<string, UriAndCid> = {}
   for (const create of verifiedRepo.creates) {
     const uri = AtUri.make(did, create.collection, create.rkey)
     records[uri.toString()] = {
       uri,
       cid: create.cid,
-      value: create.record,
     }
   }
   return records

package/src/services/indexing/plugins/block.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as Block from '../../../lexicon/types/app/bsky/graph/block'
 import * as lex from '../../../lexicon/lexicons'
@@ -27,7 +26,7 @@ const insertFn = async (
       cid: cid.toString(),
       creator: uri.host,
       subjectDid: obj.subject,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

package/src/services/indexing/plugins/feed-generator.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as FeedGenerator from '../../../lexicon/types/app/bsky/feed/generator'
 import * as lex from '../../../lexicon/lexicons'
@@ -33,7 +32,7 @@ const insertFn = async (
         ? JSON.stringify(obj.descriptionFacets)
         : undefined,
       avatarCid: obj.avatar?.ref.toString(),
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

package/src/services/indexing/plugins/follow.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as Follow from '../../../lexicon/types/app/bsky/graph/follow'
 import * as lex from '../../../lexicon/lexicons'
@@ -28,7 +27,7 @@ const insertFn = async (
       cid: cid.toString(),
       creator: uri.host,
       subjectDid: obj.subject,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

package/src/services/indexing/plugins/like.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as Like from '../../../lexicon/types/app/bsky/feed/like'
 import * as lex from '../../../lexicon/lexicons'
@@ -29,7 +28,7 @@ const insertFn = async (
       creator: uri.host,
       subject: obj.subject.uri,
       subjectCid: obj.subject.cid,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

package/src/services/indexing/plugins/list-block.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as ListBlock from '../../../lexicon/types/app/bsky/graph/listblock'
 import * as lex from '../../../lexicon/lexicons'
@@ -27,7 +26,7 @@ const insertFn = async (
       cid: cid.toString(),
       creator: uri.host,
       subjectUri: obj.subject,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

package/src/services/indexing/plugins/list-item.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as ListItem from '../../../lexicon/types/app/bsky/graph/listitem'
 import * as lex from '../../../lexicon/lexicons'
@@ -35,7 +34,7 @@ const insertFn = async (
       creator: uri.host,
       subjectDid: obj.subject,
       listUri: obj.list,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

package/src/services/indexing/plugins/list.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as List from '../../../lexicon/types/app/bsky/graph/list'
 import * as lex from '../../../lexicon/lexicons'
@@ -33,7 +32,7 @@ const insertFn = async (
         ? JSON.stringify(obj.descriptionFacets)
         : undefined,
       avatarCid: obj.avatar?.ref.toString(),
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

package/src/services/indexing/plugins/post.ts

@@ -1,7 +1,6 @@
 import { Insertable, Selectable, sql } from 'kysely'
 import { CID } from 'multiformats/cid'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { jsonStringToLex } from '@atproto/lexicon'
 import {
   Record as PostRecord,
@@ -68,7 +67,7 @@ const insertFn = async (
     cid: cid.toString(),
     creator: uri.host,
     text: obj.text,
-    createdAt: toSimplifiedISOSafe(obj.createdAt),
+    createdAt: normalizeDatetimeAlways(obj.createdAt),
     replyRoot: obj.reply?.root?.uri || null,
     replyRootCid: obj.reply?.root?.cid || null,
     replyParent: obj.reply?.parent?.uri || null,
@@ -113,6 +112,7 @@ const insertFn = async (
       obj.reply,
     )
     if (invalidReplyRoot || violatesThreadGate) {
+      Object.assign(insertedPost, { invalidReplyRoot, violatesThreadGate })
       await db
         .updateTable('post')
         .where('uri', '=', post.uri)
@@ -242,6 +242,13 @@ const notifsForInsert = (obj: IndexedPost) => {
     }
   }
 
+  if (obj.post.violatesThreadGate) {
+    // don't generate reply notifications when post violates threadgate
+    return notifs
+  }
+
+  // reply notifications
+
   for (const ancestor of obj.ancestors ?? []) {
     if (ancestor.uri === obj.post.uri) continue // no need to notify for own post
     if (ancestor.height < REPLY_NOTIF_DEPTH) {
@@ -354,6 +361,11 @@ const updateAggregates = async (db: DatabaseSchema, postIdx: IndexedPost) => {
           replyCount: db
             .selectFrom('post')
             .where('post.replyParent', '=', postIdx.post.replyParent)
+            .where((qb) =>
+              qb
+                .where('post.violatesThreadGate', 'is', null)
+                .orWhere('post.violatesThreadGate', '=', false),
+            )
             .select(countAll.as('count')),
         })
         .onConflict((oc) =>
@@ -420,7 +432,7 @@ async function validateReply(
   const violatesThreadGate = await feedutil.violatesThreadGate(
     db,
     creator,
-    new AtUri(reply.root.uri).host,
+    new AtUri(reply.root.uri).hostname,
     replyRefs.root?.record ?? null,
     replyRefs.gate?.record ?? null,
   )

package/src/services/indexing/plugins/repost.ts

@@ -1,7 +1,6 @@
 import { Selectable } from 'kysely'
 import { CID } from 'multiformats/cid'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import * as Repost from '../../../lexicon/types/app/bsky/feed/repost'
 import * as lex from '../../../lexicon/lexicons'
 import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
@@ -27,7 +26,7 @@ const insertFn = async (
     creator: uri.host,
     subject: obj.subject.uri,
     subjectCid: obj.subject.cid,
-    createdAt: toSimplifiedISOSafe(obj.createdAt),
+    createdAt: normalizeDatetimeAlways(obj.createdAt),
     indexedAt: timestamp,
   }
   const [inserted] = await Promise.all([

package/src/services/indexing/plugins/thread-gate.ts

@@ -1,6 +1,5 @@
-import { AtUri } from '@atproto/syntax'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { InvalidRequestError } from '@atproto/xrpc-server'
-import { toSimplifiedISOSafe } from '@atproto/common'
 import { CID } from 'multiformats/cid'
 import * as Threadgate from '../../../lexicon/types/app/bsky/feed/threadgate'
 import * as lex from '../../../lexicon/lexicons'
@@ -33,7 +32,7 @@ const insertFn = async (
       cid: cid.toString(),
       creator: uri.host,
       postUri: obj.post,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

package/src/services/label/index.ts

@@ -1,18 +1,38 @@
 import { sql } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { Database } from '../../db'
 import { Label, isSelfLabels } from '../../lexicon/types/com/atproto/label/defs'
 import { ids } from '../../lexicon/lexicons'
-import { LabelCache } from '../../label-cache'
+import { ReadThroughCache } from '../../cache/read-through'
+import { Redis } from '../../redis'
 
 export type Labels = Record<string, Label[]>
 
+export type LabelCacheOpts = {
+  redis: Redis
+  staleTTL: number
+  maxTTL: number
+}
+
 export class LabelService {
-  constructor(public db: Database, public cache: LabelCache | null) {}
+  public cache: ReadThroughCache<Label[]> | null
 
-  static creator(cache: LabelCache | null) {
-    return (db: Database) => new LabelService(db, cache)
+  constructor(public db: Database, cacheOpts: LabelCacheOpts | null) {
+    if (cacheOpts) {
+      this.cache = new ReadThroughCache(cacheOpts.redis, {
+        ...cacheOpts,
+        fetchMethod: async (subject: string) => {
+          const res = await fetchLabelsForSubjects(db, [subject])
+          return res[subject] ?? []
+        },
+        fetchManyMethod: (subjects: string[]) =>
+          fetchLabelsForSubjects(db, subjects),
+      })
+    }
+  }
+
+  static creator(cacheOpts: LabelCacheOpts | null) {
+    return (db: Database) => new LabelService(db, cacheOpts)
   }
 
   async formatAndCreate(
@@ -73,24 +93,19 @@ export class LabelService {
     },
   ): Promise<Labels> {
     if (subjects.length < 1) return {}
-    const res =
-      this.cache === null || opts?.skipCache
-        ? await this.db.db
-            .selectFrom('label')
-            .where('label.uri', 'in', subjects)
-            .if(!opts?.includeNeg, (qb) => qb.where('neg', '=', false))
-            .selectAll()
-            .execute()
-        : this.cache.forSubjects(subjects, opts?.includeNeg)
-    return res.reduce((acc, cur) => {
-      acc[cur.uri] ??= []
-      acc[cur.uri].push({
-        ...cur,
-        cid: cur.cid === '' ? undefined : cur.cid,
-        neg: cur.neg,
-      })
-      return acc
-    }, {} as Labels)
+    const res = this.cache
+      ? await this.cache.getMany(subjects, { revalidate: opts?.skipCache })
+      : await fetchLabelsForSubjects(this.db, subjects)
+
+    if (opts?.includeNeg) {
+      return res
+    }
+
+    const noNegs: Labels = {}
+    for (const [key, val] of Object.entries(res)) {
+      noNegs[key] = val.filter((label) => !label.neg)
+    }
+    return noNegs
   }
 
   // gets labels for any record. when did is present, combine labels for both did & profile record.
@@ -166,9 +181,37 @@ export function getSelfLabels(details: {
   const src = new AtUri(uri).host // record creator
   const cts =
     typeof record.createdAt === 'string'
-      ? toSimplifiedISOSafe(record.createdAt)
+      ? normalizeDatetimeAlways(record.createdAt)
       : new Date(0).toISOString()
   return record.labels.values.map(({ val }) => {
     return { src, uri, cid, val, cts, neg: false }
   })
 }
+
+const fetchLabelsForSubjects = async (
+  db: Database,
+  subjects: string[],
+): Promise<Record<string, Label[]>> => {
+  if (subjects.length === 0) {
+    return {}
+  }
+  const res = await db.db
+    .selectFrom('label')
+    .where('label.uri', 'in', subjects)
+    .selectAll()
+    .execute()
+  const labelMap = res.reduce((acc, cur) => {
+    acc[cur.uri] ??= []
+    acc[cur.uri].push({
+      ...cur,
+      cid: cur.cid === '' ? undefined : cur.cid,
+      neg: cur.neg,
+    })
+    return acc
+  }, {} as Record<string, Label[]>)
+  // ensure we cache negatives
+  for (const subject of subjects) {
+    labelMap[subject] ??= []
+  }
+  return labelMap
+}