@atproto/pds 0.4.215 → 0.4.217

package/src/actor-store/blob/transactor.ts

@@ -1,30 +1,29 @@
 import crypto from 'node:crypto'
 import stream from 'node:stream'
-import bytes from 'bytes'
 import { fromStream as fileTypeFromStream } from 'file-type'
-import { CID } from 'multiformats/cid'
 import PQueue from 'p-queue'
+import { SECOND, cloneStream, streamSize } from '@atproto/common'
 import {
-  SECOND,
-  cloneStream,
-  sha256RawToCid,
-  streamSize,
-} from '@atproto/common'
-import { BlobRef } from '@atproto/lexicon'
+  BlobRef,
+  Cid,
+  LegacyBlobRef,
+  cidForRawHash,
+  parseCid,
+} from '@atproto/lex-data'
 import { BlobNotFoundError, BlobStore, WriteOpAction } from '@atproto/repo'
-import { AtUri } from '@atproto/syntax'
+import { AtUri, currentDatetimeString } from '@atproto/syntax'
 import { InvalidRequestError } from '@atproto/xrpc-server'
 import { BackgroundQueue } from '../../background'
-import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
+import { com } from '../../lexicons/index.js'
 import { blobStoreLogger as log } from '../../logger'
-import { PreparedBlobRef, PreparedWrite } from '../../repo/types'
+import { PreparedWrite } from '../../repo/types'
 import { ActorDb, Blob as BlobTable } from '../db'
 import { BlobReader } from './reader'
 
 export type BlobMetadata = {
   tempKey: string
   size: number
-  cid: CID
+  cid: Cid
   mimeType: string
 }
 
@@ -37,10 +36,13 @@ export class BlobTransactor extends BlobReader {
     super(db, blobstore)
   }
 
-  async insertBlobs(recordUri: string, blobs: Iterable<BlobRef>) {
-    const values = Array.from(blobs, (cid) => ({
+  async insertBlobs(
+    recordUri: string,
+    blobs: Iterable<BlobRef | LegacyBlobRef>,
+  ) {
+    const values = Array.from(blobs, (blob) => ({
       recordUri,
-      blobCid: cid.ref.toString(),
+      blobCid: blobCid(blob).toString(),
     }))
 
     if (values.length) {
@@ -63,23 +65,22 @@ export class BlobTransactor extends BlobReader {
       mimeTypeFromStream(cloneStream(blobStream)),
     ])
 
-    const cid = sha256RawToCid(sha256)
-    const mimeType = sniffedMime || userSuggestedMime
-
     return {
       tempKey,
       size,
-      cid,
-      mimeType,
+      cid: cidForRawHash(sha256),
+      mimeType: sniffedMime || userSuggestedMime,
     }
   }
 
-  async trackUntetheredBlob(metadata: BlobMetadata) {
+  async trackUntetheredBlob(metadata: BlobMetadata): Promise<BlobRef> {
     const { tempKey, size, cid, mimeType } = metadata
+    const cidStr = cid.toString()
+
     const found = await this.db.db
       .selectFrom('blob')
       .selectAll()
-      .where('cid', '=', cid.toString())
+      .where('cid', '=', cidStr)
       .executeTakeFirst()
     if (found?.takedownRef) {
       throw new InvalidRequestError('Blob has been takendown, cannot re-upload')
@@ -88,11 +89,11 @@ export class BlobTransactor extends BlobReader {
     await this.db.db
       .insertInto('blob')
       .values({
-        cid: cid.toString(),
+        cid: cidStr,
         mimeType,
         size,
         tempKey,
-        createdAt: new Date().toISOString(),
+        createdAt: currentDatetimeString(),
       })
       .onConflict((oc) =>
         oc
@@ -101,7 +102,13 @@ export class BlobTransactor extends BlobReader {
           .where('blob.tempKey', 'is not', null),
       )
       .execute()
-    return new BlobRef(cid, mimeType, size)
+
+    return {
+      $type: 'blob',
+      ref: cid,
+      mimeType,
+      size,
+    }
   }
 
   async processWriteBlobs(rev: string, writes: PreparedWrite[]) {
@@ -143,9 +150,12 @@ export class BlobTransactor extends BlobReader {
     }
   }
 
-  async updateBlobTakedownStatus(cid: CID, takedown: StatusAttr) {
+  async updateBlobTakedownStatus(
+    cid: Cid,
+    takedown: com.atproto.admin.defs.StatusAttr,
+  ) {
     const takedownRef = takedown.applied
-      ? takedown.ref ?? new Date().toISOString()
+      ? takedown.ref ?? currentDatetimeString()
       : null
     await this.db.db
       .updateTable('blob')
@@ -199,7 +209,7 @@ export class BlobTransactor extends BlobReader {
 
     const newBlobCids = writes
       .filter((w) => isUpdate(w) || isCreate(w))
-      .flatMap((w) => w.blobs.map((b) => b.cid.toString()))
+      .flatMap((w) => w.blobs.map((b) => b.ref.toString()))
 
     const cidsToKeep = [
       ...newBlobCids,
@@ -220,7 +230,7 @@ export class BlobTransactor extends BlobReader {
       this.db.onCommit(() => {
         this.backgroundQueue.add(async () => {
           try {
-            const cids = cidsToDelete.map((cid) => CID.parse(cid))
+            const cids = cidsToDelete.map((cid) => parseCid(cid))
             await this.blobstore.deleteMany(cids)
           } catch (err) {
             log.error(
@@ -234,13 +244,13 @@ export class BlobTransactor extends BlobReader {
   }
 
   async verifyBlobAndMakePermanent(
-    blob: PreparedBlobRef,
+    blob: BlobRef,
     signal?: AbortSignal,
   ): Promise<void> {
     const found = await this.db.db
       .selectFrom('blob')
       .select(['tempKey', 'size', 'mimeType'])
-      .where('cid', '=', blob.cid.toString())
+      .where('cid', '=', blob.ref.toString())
       .where('takedownRef', 'is', null)
       .executeTakeFirst()
 
@@ -248,7 +258,7 @@ export class BlobTransactor extends BlobReader {
 
     if (!found) {
       throw new InvalidRequestError(
-        `Could not find blob: ${blob.cid.toString()}`,
+        `Could not find blob: ${blob.ref.toString()}`,
         'BlobNotFound',
       )
     }
@@ -265,10 +275,10 @@ export class BlobTransactor extends BlobReader {
       // transaction.
 
       await this.blobstore
-        .makePermanent(found.tempKey, blob.cid)
+        .makePermanent(found.tempKey, blob.ref)
         .catch((err) => {
           log.error(
-            { err, cid: blob.cid.toString() },
+            { err, cid: blob.ref.toString() },
             'could not make blob permanent',
           )
 
@@ -285,24 +295,24 @@ export class BlobTransactor extends BlobReader {
     }
   }
 
-  async insertBlobMetadata(blob: PreparedBlobRef): Promise<void> {
+  async insertBlobMetadata(blob: BlobRef): Promise<void> {
     await this.db.db
       .insertInto('blob')
       .values({
-        cid: blob.cid.toString(),
+        cid: blob.ref.toString(),
         mimeType: blob.mimeType,
         size: blob.size,
-        createdAt: new Date().toISOString(),
+        createdAt: currentDatetimeString(),
       })
       .onConflict((oc) => oc.doNothing())
       .execute()
   }
 
-  async associateBlob(blob: PreparedBlobRef, recordUri: AtUri): Promise<void> {
+  async associateBlob(blob: BlobRef, recordUri: AtUri): Promise<void> {
     await this.db.db
       .insertInto('record_blob')
       .values({
-        blobCid: blob.cid.toString(),
+        blobCid: blob.ref.toString(),
         recordUri: recordUri.toString(),
       })
       .onConflict((oc) => oc.doNothing())
@@ -311,8 +321,8 @@ export class BlobTransactor extends BlobReader {
 }
 
 export class CidNotFound extends Error {
-  cid: CID
-  constructor(cid: CID) {
+  cid: Cid
+  constructor(cid: Cid) {
     super(`cid not found: ${cid.toString()}`)
     this.cid = cid
   }
@@ -340,46 +350,24 @@ async function mimeTypeFromStream(
   return fileType?.mime
 }
 
-function acceptedMime(mime: string, accepted: string[]): boolean {
-  if (accepted.includes('*/*')) return true
-  const globs = accepted.filter((a) => a.endsWith('/*'))
-  for (const glob of globs) {
-    const [start] = glob.split('/')
-    if (mime.startsWith(`${start}/`)) {
-      return true
-    }
-  }
-  return accepted.includes(mime)
-}
-
+/**
+ * Ensures that the blob referenced in the record matches the stored blob.
+ */
 function verifyBlob(
-  blob: PreparedBlobRef,
+  blob: BlobRef,
   found: Pick<BlobTable, 'size' | 'mimeType'>,
 ) {
-  const throwInvalid = (msg: string, errName = 'InvalidBlob') => {
-    throw new InvalidRequestError(msg, errName)
-  }
-  if (blob.constraints.maxSize && found.size > blob.constraints.maxSize) {
-    throwInvalid(
-      `This file is too large. It is ${bytes.format(
-        found.size,
-      )} but the maximum size is ${bytes.format(blob.constraints.maxSize)}.`,
-      'BlobTooLarge',
-    )
-  }
   if (blob.mimeType !== found.mimeType) {
-    throwInvalid(
+    throw new InvalidRequestError(
       `Referenced Mimetype does not match stored blob. Expected: ${found.mimeType}, Got: ${blob.mimeType}`,
       'InvalidMimeType',
     )
   }
-  if (
-    blob.constraints.accept &&
-    !acceptedMime(blob.mimeType, blob.constraints.accept)
-  ) {
-    throwInvalid(
-      `Wrong type of file. It is ${blob.mimeType} but it must match ${blob.constraints.accept}.`,
-      'InvalidMimeType',
+
+  if (blob.size !== found.size) {
+    throw new InvalidRequestError(
+      `Referenced Size does not match stored blob. Expected: ${found.size}, Got: ${blob.size}`,
+      'InvalidSize',
     )
   }
 }
@@ -393,3 +381,7 @@ function isUpdate(write: PreparedWrite) {
 function isDelete(write: PreparedWrite) {
   return write.action === WriteOpAction.Delete
 }
+
+function blobCid(blob: BlobRef | LegacyBlobRef): Cid {
+  return '$type' in blob ? blob.ref : parseCid(blob.cid)
+}

package/src/actor-store/preference/reader.ts

@@ -1,11 +1,6 @@
+import { app } from '../../lexicons/index.js'
 import { ActorDb } from '../db'
-import {
-  DECLARED_AGE_PREF,
-  PERSONAL_DETAILS_PREF,
-  PrefAllowedOptions,
-  getAgeFromDatestring,
-  prefAllowed,
-} from './util'
+import { PrefAllowedOptions, getAgeFromDatestring, prefAllowed } from './util'
 
 export class PreferenceReader {
   constructor(public db: ActorDb) {}
@@ -24,27 +19,25 @@ export class PreferenceReader {
       .filter((pref) => !namespace || prefMatchNamespace(namespace, pref.name))
       .map((pref) => JSON.parse(pref.valueJson) as AccountPreference)
     const personalDetailsPref = prefs.find(
-      (pref) => pref.$type === PERSONAL_DETAILS_PREF,
+      app.bsky.actor.defs.personalDetailsPref.$isTypeOf,
     )
 
-    if (personalDetailsPref) {
-      if (typeof personalDetailsPref.birthDate === 'string') {
-        const age = getAgeFromDatestring(personalDetailsPref.birthDate)
-        const declaredAgePref: AccountPreference = {
-          $type: DECLARED_AGE_PREF,
+    if (personalDetailsPref?.birthDate) {
+      const age = getAgeFromDatestring(personalDetailsPref.birthDate)
+      prefs.push(
+        app.bsky.actor.defs.declaredAgePref.$build({
           isOverAge13: age >= 13,
           isOverAge16: age >= 16,
           isOverAge18: age >= 18,
-        }
-        prefs.push(declaredAgePref)
-      }
+        }),
+      )
     }
 
     return prefs.filter((pref) => prefAllowed(pref.$type, opts))
   }
 }
 
-export type AccountPreference = Record<string, unknown> & { $type: string }
+export type AccountPreference = app.bsky.actor.defs.Preferences[number]
 
 export const prefMatchNamespace = (namespace: string, fullname: string) => {
   return fullname === namespace || fullname.startsWith(`${namespace}.`)

package/src/actor-store/preference/util.ts

@@ -1,7 +1,4 @@
-export const PERSONAL_DETAILS_PREF = 'app.bsky.actor.defs#personalDetailsPref'
-export const DECLARED_AGE_PREF = 'app.bsky.actor.defs#declaredAgePref'
-
-const FULL_ACCESS_ONLY_PREFS = new Set([PERSONAL_DETAILS_PREF])
+import { app } from '../../lexicons/index.js'
 
 export type PrefAllowedOptions = {
   hasAccessFull?: boolean
@@ -15,13 +12,19 @@ export function prefAllowed(
     return true
   }
 
-  return !FULL_ACCESS_ONLY_PREFS.has(prefType)
+  return !isFullAccessOnlyPref(prefType)
 }
 
-const READ_ONLY_PREFS = new Set([DECLARED_AGE_PREF])
+export function isFullAccessOnlyPref(
+  type: string,
+): type is typeof app.bsky.actor.defs.personalDetailsPref.$type {
+  return type === app.bsky.actor.defs.personalDetailsPref.$type
+}
 
-export function isReadOnlyPref(prefType: string) {
-  return READ_ONLY_PREFS.has(prefType)
+export function isReadOnlyPref(
+  type: string,
+): type is typeof app.bsky.actor.defs.declaredAgePref.$type {
+  return type === app.bsky.actor.defs.declaredAgePref.$type
 }
 
 export function getAgeFromDatestring(birthDate: string): number {

package/src/actor-store/record/reader.ts

@@ -1,20 +1,22 @@
-import { CID } from 'multiformats/cid'
-import { RepoRecord } from '@atproto/lexicon'
+import { Cid, LexMap, parseCid } from '@atproto/lex-data'
 import { CidSet, cborToLexRecord, formatDataKey } from '@atproto/repo'
-import * as syntax from '@atproto/syntax'
-import { AtUri, ensureValidAtUri } from '@atproto/syntax'
+import {
+  AtUri,
+  AtUriString,
+  DatetimeString,
+  NsidString,
+  ensureValidAtUri,
+  ensureValidDid,
+} from '@atproto/syntax'
 import { countAll, notSoftDeletedClause } from '../../db/util'
-import { ids } from '../../lexicon/lexicons'
-import { Record as ProfileRecord } from '../../lexicon/types/app/bsky/actor/profile'
-import { Record as PostRecord } from '../../lexicon/types/app/bsky/feed/post'
-import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
+import { app, com } from '../../lexicons/index.js'
 import { LocalRecords } from '../../read-after-write/types'
 import { ActorDb, Backlink } from '../db'
 
 export type RecordDescript = {
   uri: string
   path: string
-  cid: CID
+  cid: Cid
 }
 
 export class RecordReader {
@@ -44,7 +46,7 @@ export class RecordReader {
         records.push({
           uri: row.uri,
           path: formatDataKey(parsed.collection, parsed.rkey),
-          cid: CID.parse(row.cid),
+          cid: parseCid(row.cid),
         })
       }
       cursor = res.at(-1)?.uri
@@ -52,14 +54,14 @@ export class RecordReader {
     return records
   }
 
-  async listCollections(): Promise<string[]> {
+  async listCollections(): Promise<NsidString[]> {
     const collections = await this.db.db
       .selectFrom('record')
       .select('collection')
       .groupBy('collection')
       .execute()
 
-    return collections.map((row) => row.collection)
+    return collections.map((row) => row.collection as NsidString)
   }
 
   async listRecordsForCollection(opts: {
@@ -70,7 +72,7 @@ export class RecordReader {
     rkeyStart?: string
     rkeyEnd?: string
     includeSoftDeleted?: boolean
-  }): Promise<{ uri: string; cid: string; value: Record<string, unknown> }[]> {
+  }): Promise<{ uri: AtUriString; cid: string; value: LexMap }[]> {
     const {
       collection,
       limit,
@@ -109,13 +111,11 @@ export class RecordReader {
       }
     }
     const res = await builder.execute()
-    return res.map((row) => {
-      return {
-        uri: row.uri,
-        cid: row.cid,
-        value: cborToLexRecord(row.content),
-      }
-    })
+    return res.map((row) => ({
+      uri: row.uri as AtUriString,
+      cid: row.cid,
+      value: cborToLexRecord(row.content),
+    }))
   }
 
   async getRecord(
@@ -125,7 +125,7 @@ export class RecordReader {
   ): Promise<{
     uri: string
     cid: string
-    value: Record<string, unknown>
+    value: LexMap
     indexedAt: string
     takedownRef: string | null
   } | null> {
@@ -172,7 +172,9 @@ export class RecordReader {
     return !!record
   }
 
-  async getRecordTakedownStatus(uri: AtUri): Promise<StatusAttr | null> {
+  async getRecordTakedownStatus(
+    uri: AtUri,
+  ): Promise<com.atproto.admin.defs.StatusAttr | null> {
     const res = await this.db.db
       .selectFrom('record')
       .select('takedownRef')
@@ -184,13 +186,13 @@ export class RecordReader {
       : { applied: false }
   }
 
-  async getCurrentRecordCid(uri: AtUri): Promise<CID | null> {
+  async getCurrentRecordCid(uri: AtUri): Promise<Cid | null> {
     const res = await this.db.db
       .selectFrom('record')
       .select('cid')
       .where('uri', '=', uri.toString())
       .executeTakeFirst()
-    return res ? CID.parse(res.cid) : null
+    return res ? parseCid(res.cid) : null
   }
 
   async getRecordBacklinks(opts: {
@@ -212,7 +214,7 @@ export class RecordReader {
   // @NOTE this logic is a placeholder until we allow users to specify these constraints themselves.
   // Ensures that we don't end-up with duplicate likes, reposts, and follows from race conditions.
 
-  async getBacklinkConflicts(uri: AtUri, record: RepoRecord): Promise<AtUri[]> {
+  async getBacklinkConflicts(uri: AtUri, record: LexMap): Promise<AtUri[]> {
     const conflicts: AtUri[] = []
 
     for (const backlink of getBacklinks(uri, record)) {
@@ -242,7 +244,7 @@ export class RecordReader {
         .limit(1000)
         .execute()
       for (const row of res) {
-        cids.add(CID.parse(row.cid))
+        cids.add(parseCid(row.cid))
       }
       cursor = res.at(-1)?.cid
     }
@@ -253,14 +255,14 @@ export class RecordReader {
     const row = await this.db.db
       .selectFrom('record')
       .leftJoin('repo_block', 'repo_block.cid', 'record.cid')
-      .where('record.collection', '=', ids.AppBskyActorProfile)
+      .where('record.collection', '=', app.bsky.actor.profile.$type)
       .where('record.rkey', '=', 'self')
       .selectAll()
       .executeTakeFirst()
 
     if (!row?.content) return null
 
-    return cborToLexRecord(row.content) as ProfileRecord
+    return cborToLexRecord(row.content) as app.bsky.actor.profile.Main
   }
 
   async getRecordsSinceRev(rev: string): Promise<LocalRecords> {
@@ -298,19 +300,22 @@ export class RecordReader {
       result.count++
 
       const uri = new AtUri(cur.uri)
-      if (uri.collection === ids.AppBskyActorProfile && uri.rkey === 'self') {
+      if (
+        uri.collection === app.bsky.actor.profile.$type &&
+        uri.rkey === 'self'
+      ) {
         result.profile = {
           uri,
-          cid: CID.parse(cur.cid),
-          indexedAt: cur.indexedAt,
-          record: cborToLexRecord(cur.content) as ProfileRecord,
+          cid: parseCid(cur.cid),
+          indexedAt: cur.indexedAt as DatetimeString,
+          record: cborToLexRecord(cur.content) as app.bsky.actor.profile.Main,
         }
-      } else if (uri.collection === ids.AppBskyFeedPost) {
+      } else if (uri.collection === app.bsky.feed.post.$type) {
         result.posts.push({
           uri,
-          cid: CID.parse(cur.cid),
-          indexedAt: cur.indexedAt,
-          record: cborToLexRecord(cur.content) as PostRecord,
+          cid: parseCid(cur.cid),
+          indexedAt: cur.indexedAt as DatetimeString,
+          record: cborToLexRecord(cur.content) as app.bsky.feed.post.Main,
         })
       }
     }
@@ -322,17 +327,17 @@ export class RecordReader {
 // @NOTE in the future this can be replaced with a more generic routine that pulls backlinks based on lex docs.
 // For now we just want to ensure we're tracking links from follows, blocks, likes, and reposts.
 
-export const getBacklinks = (uri: AtUri, record: RepoRecord): Backlink[] => {
+export const getBacklinks = (uri: AtUri, record: LexMap): Backlink[] => {
   if (
-    record?.['$type'] === ids.AppBskyGraphFollow ||
-    record?.['$type'] === ids.AppBskyGraphBlock
+    record?.['$type'] === app.bsky.graph.follow.$type ||
+    record?.['$type'] === app.bsky.graph.block.$type
   ) {
     const subject = record['subject']
     if (typeof subject !== 'string') {
       return []
     }
     try {
-      syntax.ensureValidDid(subject)
+      ensureValidDid(subject)
     } catch {
       return []
     }
@@ -345,8 +350,8 @@ export const getBacklinks = (uri: AtUri, record: RepoRecord): Backlink[] => {
     ]
   }
   if (
-    record?.['$type'] === ids.AppBskyFeedLike ||
-    record?.['$type'] === ids.AppBskyFeedRepost
+    record?.['$type'] === app.bsky.feed.like.$type ||
+    record?.['$type'] === app.bsky.feed.repost.$type
   ) {
     const subject = record['subject']
     if (typeof subject?.['uri'] !== 'string') {

package/src/actor-store/record/transactor.ts

@@ -1,8 +1,7 @@
-import { CID } from 'multiformats/cid'
-import { RepoRecord } from '@atproto/lexicon'
+import { Cid, LexMap } from '@atproto/lex-data'
 import { BlobStore, WriteOpAction } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
-import { StatusAttr } from '../../lexicon/types/com/atproto/admin/defs'
+import { com } from '../../lexicons/index.js'
 import { dbLogger as log } from '../../logger'
 import { ActorDb, Backlink } from '../db'
 import { RecordReader, getBacklinks } from './reader'
@@ -17,8 +16,8 @@ export class RecordTransactor extends RecordReader {
 
   async indexRecord(
     uri: AtUri,
-    cid: CID,
-    record: RepoRecord | null,
+    cid: Cid,
+    record: LexMap | null,
     action: WriteOpAction.Create | WriteOpAction.Update = WriteOpAction.Create,
     repoRev: string,
     timestamp?: string,
@@ -96,7 +95,10 @@ export class RecordTransactor extends RecordReader {
       .execute()
   }
 
-  async updateRecordTakedownStatus(uri: AtUri, takedown: StatusAttr) {
+  async updateRecordTakedownStatus(
+    uri: AtUri,
+    takedown: com.atproto.admin.defs.StatusAttr,
+  ) {
     const takedownRef = takedown.applied
       ? takedown.ref ?? new Date().toISOString()
       : null