@atproto/pds 0.5.11 → 0.5.14

package/src/actor-store/blob/transactor.ts

@@ -1,383 +0,0 @@
-import crypto from 'node:crypto'
-import stream from 'node:stream'
-// eslint-disable-next-line import/default, import/no-named-as-default-member
-import fileType from 'file-type'
-const { fromStream: fileTypeFromStream } = fileType
-import PQueue from 'p-queue'
-import { SECOND, cloneStream, streamSize } from '@atproto/common'
-import {
-  BlobRef,
-  Cid,
-  TypedBlobRef,
-  cidForRawHash,
-  getBlobCidString,
-  parseCid,
-} from '@atproto/lex-data'
-import { BlobNotFoundError, BlobStore, WriteOpAction } from '@atproto/repo'
-import { AtUri, currentDatetimeString } from '@atproto/syntax'
-import { InvalidRequestError } from '@atproto/xrpc-server'
-import { BackgroundQueue } from '../../background.js'
-import { com } from '../../lexicons/index.js'
-import { blobStoreLogger as log } from '../../logger.js'
-import { PreparedWrite } from '../../repo/types.js'
-import { ActorDb, Blob as BlobTable } from '../db/index.js'
-import { BlobReader } from './reader.js'
-
-export type BlobMetadata = {
-  tempKey: string
-  size: number
-  cid: Cid
-  mimeType: string
-}
-
-export class BlobTransactor extends BlobReader {
-  constructor(
-    public db: ActorDb,
-    public blobstore: BlobStore,
-    public backgroundQueue: BackgroundQueue,
-  ) {
-    super(db, blobstore)
-  }
-
-  async insertBlobs(recordUri: string, blobs: Iterable<BlobRef>) {
-    const values = Array.from(blobs, (blob) => ({
-      recordUri,
-      blobCid: getBlobCidString(blob),
-    }))
-
-    if (values.length) {
-      await this.db.db
-        .insertInto('record_blob')
-        .values(values)
-        .onConflict((oc) => oc.doNothing())
-        .execute()
-    }
-  }
-
-  async uploadBlobAndGetMetadata(
-    userSuggestedMime: string,
-    blobStream: stream.Readable,
-  ): Promise<BlobMetadata> {
-    const [tempKey, size, sha256, sniffedMime] = await Promise.all([
-      this.blobstore.putTemp(cloneStream(blobStream)),
-      streamSize(cloneStream(blobStream)),
-      sha256Stream(cloneStream(blobStream)),
-      mimeTypeFromStream(cloneStream(blobStream)),
-    ])
-
-    return {
-      tempKey,
-      size,
-      cid: cidForRawHash(sha256),
-      mimeType: sniffedMime || userSuggestedMime,
-    }
-  }
-
-  async trackUntetheredBlob(metadata: BlobMetadata): Promise<TypedBlobRef> {
-    const { tempKey, size, cid, mimeType } = metadata
-    const cidStr = cid.toString()
-
-    const found = await this.db.db
-      .selectFrom('blob')
-      .selectAll()
-      .where('cid', '=', cidStr)
-      .executeTakeFirst()
-    if (found?.takedownRef) {
-      throw new InvalidRequestError('Blob has been takendown, cannot re-upload')
-    }
-
-    await this.db.db
-      .insertInto('blob')
-      .values({
-        cid: cidStr,
-        mimeType,
-        size,
-        tempKey,
-        createdAt: currentDatetimeString(),
-      })
-      .onConflict((oc) =>
-        oc
-          .column('cid')
-          .doUpdateSet({ tempKey })
-          .where('blob.tempKey', 'is not', null),
-      )
-      .execute()
-
-    return {
-      $type: 'blob',
-      ref: cid,
-      mimeType,
-      size,
-    }
-  }
-
-  async processWriteBlobs(rev: string, writes: PreparedWrite[]) {
-    await this.deleteDereferencedBlobs(writes)
-
-    const ac = new AbortController()
-
-    // Limit the number of parallel requests made to the BlobStore by using a
-    // a queue with concurrency management.
-    type Task = () => Promise<void>
-    const tasks: Task[] = []
-
-    for (const write of writes) {
-      if (isCreate(write) || isUpdate(write)) {
-        for (const blob of write.blobs) {
-          tasks.push(async () => {
-            if (ac.signal.aborted) return
-            await this.associateBlob(blob, write.uri)
-            await this.verifyBlobAndMakePermanent(blob, ac.signal)
-          })
-        }
-      }
-    }
-
-    try {
-      const queue = new PQueue({
-        concurrency: 20,
-        // The blob store should already limit the time of every operation. We
-        // add a timeout here as an extra precaution.
-        timeout: 60 * SECOND,
-        throwOnTimeout: true,
-      })
-
-      // Will reject as soon as any task fails, causing the "finally" block
-      // below to run, aborting every other pending tasks.
-      await queue.addAll(tasks)
-    } finally {
-      ac.abort()
-    }
-  }
-
-  async updateBlobTakedownStatus(
-    cid: Cid,
-    takedown: com.atproto.admin.defs.StatusAttr,
-  ) {
-    const takedownRef = takedown.applied
-      ? takedown.ref ?? currentDatetimeString()
-      : null
-    await this.db.db
-      .updateTable('blob')
-      .set({ takedownRef })
-      .where('cid', '=', cid.toString())
-      .executeTakeFirst()
-
-    try {
-      // @NOTE find a way to not perform i/o operations during the transaction
-      // (typically by using a state in the "blob" table, and another process to
-      // handle the actual i/o)
-      if (takedown.applied) {
-        await this.blobstore.quarantine(cid)
-      } else {
-        await this.blobstore.unquarantine(cid)
-      }
-    } catch (err) {
-      if (!(err instanceof BlobNotFoundError)) {
-        log.error(
-          { err, cid: cid.toString() },
-          'could not update blob takedown status',
-        )
-
-        throw err
-      }
-    }
-  }
-
-  async deleteDereferencedBlobs(
-    writes: PreparedWrite[],
-    skipBlobStore?: boolean,
-  ) {
-    const deletes = writes.filter(isDelete)
-    const updates = writes.filter(isUpdate)
-    const uris = [...deletes, ...updates].map((w) => w.uri.toString())
-    if (uris.length === 0) return
-
-    const deletedRepoBlobs = await this.db.db
-      .deleteFrom('record_blob')
-      .where('recordUri', 'in', uris)
-      .returning('blobCid')
-      .execute()
-    if (deletedRepoBlobs.length === 0) return
-
-    const deletedRepoBlobCids = deletedRepoBlobs.map((row) => row.blobCid)
-    const duplicateCids = await this.db.db
-      .selectFrom('record_blob')
-      .where('blobCid', 'in', deletedRepoBlobCids)
-      .select('blobCid')
-      .execute()
-
-    const newBlobCids = writes
-      .filter((w) => isUpdate(w) || isCreate(w))
-      .flatMap((w) => w.blobs.map((b) => b.ref.toString()))
-
-    const cidsToKeep = [
-      ...newBlobCids,
-      ...duplicateCids.map((row) => row.blobCid),
-    ]
-
-    const cidsToDelete = deletedRepoBlobCids.filter(
-      (cid) => !cidsToKeep.includes(cid),
-    )
-    if (cidsToDelete.length === 0) return
-
-    await this.db.db
-      .deleteFrom('blob')
-      .where('cid', 'in', cidsToDelete)
-      .execute()
-
-    if (!skipBlobStore) {
-      this.db.onCommit(() => {
-        this.backgroundQueue.add(async () => {
-          try {
-            const cids = cidsToDelete.map((cid) => parseCid(cid))
-            await this.blobstore.deleteMany(cids)
-          } catch (err) {
-            log.error(
-              { err, cids: cidsToDelete },
-              'could not delete blobs from blobstore',
-            )
-          }
-        })
-      })
-    }
-  }
-
-  async verifyBlobAndMakePermanent(
-    blob: TypedBlobRef,
-    signal?: AbortSignal,
-  ): Promise<void> {
-    const found = await this.db.db
-      .selectFrom('blob')
-      .select(['tempKey', 'size', 'mimeType'])
-      .where('cid', '=', blob.ref.toString())
-      .where('takedownRef', 'is', null)
-      .executeTakeFirst()
-
-    signal?.throwIfAborted()
-
-    if (!found) {
-      throw new InvalidRequestError(
-        `Could not find blob: ${blob.ref.toString()}`,
-        'BlobNotFound',
-      )
-    }
-
-    if (found.tempKey) {
-      verifyBlob(blob, found)
-
-      // @NOTE it is less than ideal to perform async (i/o) operations during a
-      // transaction. Especially since there have been instances of the actor-db
-      // being locked, requiring to kick the processes.
-
-      // The better solution would be to update the blob state in the database
-      // (e.g. "makeItPermanent") and to process those updates outside of the
-      // transaction.
-
-      await this.blobstore
-        .makePermanent(found.tempKey, blob.ref)
-        .catch((err) => {
-          log.error(
-            { err, cid: blob.ref.toString() },
-            'could not make blob permanent',
-          )
-
-          throw err
-        })
-
-      signal?.throwIfAborted()
-
-      await this.db.db
-        .updateTable('blob')
-        .set({ tempKey: null })
-        .where('tempKey', '=', found.tempKey)
-        .execute()
-    }
-  }
-
-  async insertBlobMetadata(blob: TypedBlobRef): Promise<void> {
-    await this.db.db
-      .insertInto('blob')
-      .values({
-        cid: blob.ref.toString(),
-        mimeType: blob.mimeType,
-        size: blob.size,
-        createdAt: currentDatetimeString(),
-      })
-      .onConflict((oc) => oc.doNothing())
-      .execute()
-  }
-
-  async associateBlob(blob: TypedBlobRef, recordUri: AtUri): Promise<void> {
-    await this.db.db
-      .insertInto('record_blob')
-      .values({
-        blobCid: blob.ref.toString(),
-        recordUri: recordUri.toString(),
-      })
-      .onConflict((oc) => oc.doNothing())
-      .execute()
-  }
-}
-
-export class CidNotFound extends Error {
-  cid: Cid
-  constructor(cid: Cid) {
-    super(`cid not found: ${cid.toString()}`)
-    this.cid = cid
-  }
-}
-
-async function sha256Stream(toHash: stream.Readable): Promise<Uint8Array> {
-  const hash = crypto.createHash('sha256')
-  try {
-    for await (const chunk of toHash) {
-      hash.write(chunk)
-    }
-  } catch (err) {
-    hash.end()
-    throw err
-  }
-  hash.end()
-  return hash.read()
-}
-
-async function mimeTypeFromStream(
-  blobStream: stream.Readable,
-): Promise<string | undefined> {
-  const fileType = await fileTypeFromStream(blobStream)
-  blobStream.destroy()
-  return fileType?.mime
-}
-
-/**
- * Ensures that the blob referenced in the record matches the stored blob.
- */
-function verifyBlob(
-  blob: TypedBlobRef,
-  found: Pick<BlobTable, 'size' | 'mimeType'>,
-) {
-  if (blob.mimeType !== found.mimeType) {
-    throw new InvalidRequestError(
-      `Referenced Mimetype does not match stored blob. Expected: ${found.mimeType}, Got: ${blob.mimeType}`,
-      'InvalidMimeType',
-    )
-  }
-
-  if (blob.size !== found.size) {
-    throw new InvalidRequestError(
-      `Referenced Size does not match stored blob. Expected: ${found.size}, Got: ${blob.size}`,
-      'InvalidSize',
-    )
-  }
-}
-
-function isCreate(write: PreparedWrite) {
-  return write.action === WriteOpAction.Create
-}
-function isUpdate(write: PreparedWrite) {
-  return write.action === WriteOpAction.Update
-}
-function isDelete(write: PreparedWrite) {
-  return write.action === WriteOpAction.Delete
-}

package/src/actor-store/db/index.ts

@@ -1,20 +0,0 @@
-import { Database, Migrator } from '../../db/index.js'
-import migrations from './migrations/index.js'
-import { DatabaseSchema } from './schema/index.js'
-export * from './schema/index.js'
-
-export type ActorDb = Database<DatabaseSchema>
-
-export const getDb = (
-  location: string,
-  disableWalAutoCheckpoint = false,
-): ActorDb => {
-  const pragmas: Record<string, string> = disableWalAutoCheckpoint
-    ? { wal_autocheckpoint: '0' }
-    : {}
-  return Database.sqlite(location, { pragmas })
-}
-
-export const getMigrator = (db: Database<DatabaseSchema>) => {
-  return new Migrator(db.db, migrations)
-}

package/src/actor-store/db/migrations/001-init.ts

@@ -1,105 +0,0 @@
-import { Kysely } from 'kysely'
-
-export async function up(db: Kysely<unknown>): Promise<void> {
-  await db.schema
-    .createTable('repo_root')
-    .addColumn('did', 'varchar', (col) => col.primaryKey())
-    .addColumn('cid', 'varchar', (col) => col.notNull())
-    .addColumn('rev', 'varchar', (col) => col.notNull())
-    .addColumn('indexedAt', 'varchar', (col) => col.notNull())
-    .execute()
-
-  await db.schema
-    .createTable('repo_block')
-    .addColumn('cid', 'varchar', (col) => col.primaryKey())
-    .addColumn('repoRev', 'varchar', (col) => col.notNull())
-    .addColumn('size', 'integer', (col) => col.notNull())
-    .addColumn('content', 'blob', (col) => col.notNull())
-    .execute()
-
-  await db.schema
-    .createIndex('repo_block_repo_rev_idx')
-    .on('repo_block')
-    .columns(['repoRev', 'cid'])
-    .execute()
-
-  await db.schema
-    .createTable('record')
-    .addColumn('uri', 'varchar', (col) => col.primaryKey())
-    .addColumn('cid', 'varchar', (col) => col.notNull())
-    .addColumn('collection', 'varchar', (col) => col.notNull())
-    .addColumn('rkey', 'varchar', (col) => col.notNull())
-    .addColumn('repoRev', 'varchar', (col) => col.notNull())
-    .addColumn('indexedAt', 'varchar', (col) => col.notNull())
-    .addColumn('takedownRef', 'varchar')
-    .execute()
-  await db.schema
-    .createIndex('record_cid_idx')
-    .on('record')
-    .column('cid')
-    .execute()
-  await db.schema
-    .createIndex('record_collection_idx')
-    .on('record')
-    .column('collection')
-    .execute()
-  await db.schema
-    .createIndex('record_repo_rev_idx')
-    .on('record')
-    .column('repoRev')
-    .execute()
-
-  await db.schema
-    .createTable('blob')
-    .addColumn('cid', 'varchar', (col) => col.primaryKey())
-    .addColumn('mimeType', 'varchar', (col) => col.notNull())
-    .addColumn('size', 'integer', (col) => col.notNull())
-    .addColumn('tempKey', 'varchar')
-    .addColumn('width', 'integer')
-    .addColumn('height', 'integer')
-    .addColumn('createdAt', 'varchar', (col) => col.notNull())
-    .addColumn('takedownRef', 'varchar')
-    .execute()
-  await db.schema
-    .createIndex('blob_tempkey_idx')
-    .on('blob')
-    .column('tempKey')
-    .execute()
-
-  await db.schema
-    .createTable('record_blob')
-    .addColumn('blobCid', 'varchar', (col) => col.notNull())
-    .addColumn('recordUri', 'varchar', (col) => col.notNull())
-    .addPrimaryKeyConstraint(`record_blob_pkey`, ['blobCid', 'recordUri'])
-    .execute()
-
-  await db.schema
-    .createTable('backlink')
-    .addColumn('uri', 'varchar', (col) => col.notNull())
-    .addColumn('path', 'varchar', (col) => col.notNull())
-    .addColumn('linkTo', 'varchar', (col) => col.notNull())
-    .addPrimaryKeyConstraint('backlinks_pkey', ['uri', 'path'])
-    .execute()
-  await db.schema
-    .createIndex('backlink_link_to_idx')
-    .on('backlink')
-    .columns(['path', 'linkTo'])
-    .execute()
-
-  await db.schema
-    .createTable('account_pref')
-    .addColumn('id', 'integer', (col) => col.autoIncrement().primaryKey())
-    .addColumn('name', 'varchar', (col) => col.notNull())
-    .addColumn('valueJson', 'text', (col) => col.notNull())
-    .execute()
-}
-
-export async function down(db: Kysely<unknown>): Promise<void> {
-  await db.schema.dropTable('account_pref').execute()
-  await db.schema.dropTable('backlink').execute()
-  await db.schema.dropTable('record_blob').execute()
-  await db.schema.dropTable('blob').execute()
-  await db.schema.dropTable('record').execute()
-  await db.schema.dropTable('repo_block').execute()
-  await db.schema.dropTable('repo_root').execute()
-}

package/src/actor-store/db/migrations/index.ts

@@ -1,5 +0,0 @@
-import * as init from './001-init.js'
-
-export default {
-  '001': init,
-}

package/src/actor-store/db/schema/account-pref.ts

@@ -1,11 +0,0 @@
-import { GeneratedAlways } from 'kysely'
-
-export interface AccountPref {
-  id: GeneratedAlways<number>
-  name: string
-  valueJson: string // json
-}
-
-export const tableName = 'account_pref'
-
-export type PartialDB = { [tableName]: AccountPref }

package/src/actor-store/db/schema/backlink.ts

@@ -1,9 +0,0 @@
-export interface Backlink {
-  uri: string
-  path: string
-  linkTo: string
-}
-
-export const tableName = 'backlink'
-
-export type PartialDB = { [tableName]: Backlink }

package/src/actor-store/db/schema/blob.ts

@@ -1,14 +0,0 @@
-export interface Blob {
-  cid: string
-  mimeType: string
-  size: number
-  tempKey: string | null
-  // width: number | null  // @TODO: actually drop these columns from the db
-  // height: number | null
-  createdAt: string
-  takedownRef: string | null
-}
-
-export const tableName = 'blob'
-
-export type PartialDB = { [tableName]: Blob }

package/src/actor-store/db/schema/index.ts

@@ -1,23 +0,0 @@
-import * as accountPref from './account-pref.js'
-import * as backlink from './backlink.js'
-import * as blob from './blob.js'
-import * as recordBlob from './record-blob.js'
-import * as record from './record.js'
-import * as repoBlock from './repo-block.js'
-import * as repoRoot from './repo-root.js'
-
-export type DatabaseSchema = accountPref.PartialDB &
-  repoRoot.PartialDB &
-  record.PartialDB &
-  backlink.PartialDB &
-  repoBlock.PartialDB &
-  blob.PartialDB &
-  recordBlob.PartialDB
-
-export type { AccountPref } from './account-pref.js'
-export type { RepoRoot } from './repo-root.js'
-export type { Record } from './record.js'
-export type { Backlink } from './backlink.js'
-export type { RepoBlock } from './repo-block.js'
-export type { Blob } from './blob.js'
-export type { RecordBlob } from './record-blob.js'

package/src/actor-store/db/schema/record-blob.ts

@@ -1,8 +0,0 @@
-export interface RecordBlob {
-  blobCid: string
-  recordUri: string
-}
-
-export const tableName = 'record_blob'
-
-export type PartialDB = { [tableName]: RecordBlob }

package/src/actor-store/db/schema/record.ts

@@ -1,14 +0,0 @@
-// @NOTE also used by app-view (moderation)
-export interface Record {
-  uri: string
-  cid: string
-  collection: string
-  rkey: string
-  repoRev: string
-  indexedAt: string
-  takedownRef: string | null
-}
-
-export const tableName = 'record'
-
-export type PartialDB = { [tableName]: Record }

package/src/actor-store/db/schema/repo-block.ts

@@ -1,10 +0,0 @@
-export interface RepoBlock {
-  cid: string
-  repoRev: string
-  size: number
-  content: Uint8Array
-}
-
-export const tableName = 'repo_block'
-
-export type PartialDB = { [tableName]: RepoBlock }

package/src/actor-store/db/schema/repo-root.ts

@@ -1,10 +0,0 @@
-export interface RepoRoot {
-  did: string
-  cid: string
-  rev: string
-  indexedAt: string
-}
-
-const tableName = 'repo_root'
-
-export type PartialDB = { [tableName]: RepoRoot }

package/src/actor-store/migrate.ts

@@ -1,39 +0,0 @@
-import { sql } from 'kysely'
-import PQueue from 'p-queue'
-import { AppContext } from '../context.js'
-
-export const forEachActorStore = async (
-  ctx: AppContext,
-  opts: { concurrency?: number },
-  fn: (ctx: AppContext, did: string) => Promise<string>,
-) => {
-  const { concurrency = 1 } = opts
-
-  const queue = new PQueue({ concurrency })
-  const actorQb = ctx.accountManager.db.db
-    .selectFrom('actor')
-    .selectAll()
-    .limit(2 * concurrency)
-  let cursor: { createdAt: string; did: string } | undefined
-  do {
-    const actors = cursor
-      ? await actorQb
-          .where(
-            sql`("createdAt", "did")`,
-            '>',
-            sql`(${cursor.createdAt}, ${cursor.did})`,
-          )
-          .execute()
-      : await actorQb.execute()
-    queue.addAll(
-      actors.map(({ did }) => {
-        return () => fn(ctx, did)
-      }),
-    )
-    cursor = actors.at(-1)
-    await queue.onEmpty() // wait for all remaining items to be in process, then move on to next page
-  } while (cursor)
-
-  // finalize remaining work
-  await queue.onIdle()
-}