@atproto/pds 0.4.176 → 0.4.177

package/src/api/com/atproto/repo/importRepo.ts

@@ -1,5 +1,4 @@
 import { CID } from 'multiformats/cid'
-import PQueue from 'p-queue'
 import { TID } from '@atproto/common'
 import { BlobRef, LexValue, RepoRecord } from '@atproto/lexicon'
 import {
@@ -11,7 +10,6 @@ import {
 } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
 import { InvalidRequestError } from '@atproto/xrpc-server'
-import { ActorStoreTransactor } from '../../../../actor-store/actor-store-transactor'
 import { ACCESS_FULL } from '../../../../auth-scope'
 import { AppContext } from '../../../../context'
 import { Server } from '../../../../lexicon'
@@ -32,53 +30,47 @@ export default function (server: Server, ctx: AppContext) {
 
       const { did } = auth.credentials
 
-      await ctx.actorStore.transact(did, (store) =>
-        importRepo(store, input.body),
-      )
-    },
-  })
-}
+      // @NOTE process as much as we can before the transaction, in particular
+      // the reading of the body stream.
+      const { roots, blocks } = await readCarStream(input.body)
+      if (roots.length !== 1) {
+        await blocks.dump()
+        throw new InvalidRequestError('expected one root')
+      }
 
-const importRepo = async (
-  actorStore: ActorStoreTransactor,
-  incomingCar: AsyncIterable<Uint8Array>,
-) => {
-  const now = new Date().toISOString()
-  const rev = TID.nextStr()
-  const did = actorStore.repo.did
+      const blockMap = new BlockMap()
+      for await (const block of blocks) {
+        blockMap.set(block.cid, block.bytes)
+      }
 
-  const { roots, blocks } = await readCarStream(incomingCar)
-  if (roots.length !== 1) {
-    await blocks.dump()
-    throw new InvalidRequestError('expected one root')
-  }
-  const blockMap = new BlockMap()
-  for await (const block of blocks) {
-    blockMap.set(block.cid, block.bytes)
-  }
-  const currRepo = await actorStore.repo.maybeLoadRepo()
-  const diff = await verifyDiff(
-    currRepo,
-    blockMap,
-    roots[0],
-    undefined,
-    undefined,
-    { ensureLeaves: false },
-  )
-  diff.commit.rev = rev
-  await actorStore.repo.storage.applyCommit(diff.commit, currRepo === null)
-  const recordQueue = new PQueue({ concurrency: 50 })
-  const controller = new AbortController()
-  for (const write of diff.writes) {
-    recordQueue
-      .add(
-        async () => {
+      await ctx.actorStore.transact(did, async (store) => {
+        const now = new Date().toISOString()
+        const rev = TID.nextStr()
+        const did = store.repo.did
+
+        const currRepo = await store.repo.maybeLoadRepo()
+        const diff = await verifyDiff(
+          currRepo,
+          blockMap,
+          roots[0],
+          undefined,
+          undefined,
+          { ensureLeaves: false },
+        )
+        diff.commit.rev = rev
+        await store.repo.storage.applyCommit(diff.commit, currRepo === null)
+
+        // @NOTE There is no point in performing the following concurrently
+        // since better-sqlite3 is synchronous.
+        for (const write of diff.writes) {
           const uri = AtUri.make(did, write.collection, write.rkey)
           if (write.action === WriteOpAction.Delete) {
-            await actorStore.record.deleteRecord(uri)
+            await store.record.deleteRecord(uri)
           } else {
             let parsedRecord: RepoRecord
             try {
+              // @NOTE getAndParseRecord returns a promise for historical
+              // reasons but it's internal processing is actually synchronous.
               const parsed = await getAndParseRecord(blockMap, write.cid)
               parsedRecord = parsed.record
             } catch {
@@ -86,7 +78,8 @@ const importRepo = async (
                 `Could not parse record at '${write.collection}/${write.rkey}'`,
               )
             }
-            const indexRecord = actorStore.record.indexRecord(
+
+            await store.record.indexRecord(
               uri,
               write.cid,
               parsedRecord,
@@ -95,19 +88,12 @@ const importRepo = async (
               now,
             )
             const recordBlobs = findBlobRefs(parsedRecord)
-            const indexRecordBlobs = actorStore.repo.blob.insertBlobs(
-              uri.toString(),
-              recordBlobs,
-            )
-            await Promise.all([indexRecord, indexRecordBlobs])
+            await store.repo.blob.insertBlobs(uri.toString(), recordBlobs)
           }
-        },
-        { signal: controller.signal },
-      )
-      .catch((err) => controller.abort(err))
-  }
-  await recordQueue.onIdle()
-  controller.signal.throwIfAborted()
+        }
+      })
+    },
+  })
 }
 
 export const findBlobRefs = (val: LexValue, layer = 0): BlobRef[] => {

package/src/auth-verifier.ts

@@ -7,7 +7,7 @@ import { IdResolver, getDidKeyFromMultibase } from '@atproto/identity'
 import {
   OAuthError,
   OAuthVerifier,
-  VerifyTokenClaimsOptions,
+  VerifyTokenPayloadOptions,
   WWWAuthenticateError,
 } from '@atproto/oauth-provider'
 import {
@@ -39,7 +39,6 @@ import {
 } from './auth-output'
 import { ACCESS_STANDARD, AuthScope, isAuthScope } from './auth-scope'
 import { softDeleted } from './db'
-import { oauthLogger } from './logger'
 import { appendVary } from './util/http'
 import { WithRequired } from './util/types'
 
@@ -338,7 +337,7 @@ export class AuthVerifier {
     OAuthOutput,
     P
   > {
-    const verifyTokenOptions: VerifyTokenClaimsOptions = {
+    const verifyTokenOptions: VerifyTokenPayloadOptions = {
       audience: [this.dids.pds],
       scope: ['atproto'],
     }
@@ -358,7 +357,7 @@ export class AuthVerifier {
       const originalUrl = req.originalUrl || req.url || '/'
       const url = new URL(originalUrl, this._publicUrl)
 
-      const { tokenClaims, dpopProof } = await this.oauthVerifier
+      const { scope, sub: did } = await this.oauthVerifier
         .authenticateRequest(
           req.method || 'GET',
           url,
@@ -383,28 +382,13 @@ export class AuthVerifier {
           throw err
         })
 
-      // @TODO drop this once oauth provider no longer accepts DPoP proof with
-      // query or fragment in "htu" claim.
-      if (dpopProof?.htu.match(/[?#]/)) {
-        oauthLogger.info(
-          {
-            client_id: tokenClaims.client_id,
-            htu: dpopProof.htu,
-          },
-          'DPoP proof "htu" contains query or fragment',
-        )
-      }
-
-      const { sub: did } = tokenClaims
       if (typeof did !== 'string' || !did.startsWith('did:')) {
         throw new InvalidRequestError('Malformed token', 'InvalidToken')
       }
 
       await this.verifyStatus(did, verifyStatusOptions)
 
-      const permissions = new ScopePermissionsTransition(
-        tokenClaims.scope?.split(' '),
-      )
+      const permissions = new ScopePermissionsTransition(scope?.split(' '))
 
       // Should never happen
       if (!permissions.scopes.has('atproto')) {

package/src/context.ts

@@ -32,6 +32,7 @@ import {
 } from '@atproto-labs/fetch-node'
 import { AccountManager } from './account-manager/account-manager'
 import { OAuthStore } from './account-manager/oauth-store'
+import { ScopeReferenceGetter } from './account-manager/scope-reference-getter'
 import { ActorStore } from './actor-store/actor-store'
 import { authPassthru, forwardedFor } from './api/proxy'
 import {
@@ -46,7 +47,7 @@ import { Crawlers } from './crawlers'
 import { DidSqliteCache } from './did-cache'
 import { DiskBlobStore } from './disk-blobstore'
 import { ImageUrlBuilder } from './image/image-url-builder'
-import { fetchLogger, lexiconResolverLogger } from './logger'
+import { fetchLogger, lexiconResolverLogger, oauthLogger } from './logger'
 import { ServerMailer } from './mailer'
 import { ModerationMailer } from './mailer/moderation'
 import { LocalViewer, LocalViewerCreator } from './read-after-write/viewer'
@@ -397,10 +398,11 @@ export class AppContext {
             protected_resources: [new URL(cfg.oauth.issuer).origin],
           },
           // If the PDS is both an authorization server & resource server (no
-          // entryway), there is no need to use JWTs as access tokens. Instead,
-          // the PDS can use tokenId as access tokens. This allows the PDS to
-          // always use up-to-date token data from the token store.
-          accessTokenMode: AccessTokenMode.light,
+          // entryway), we can afford to check the token validity on every
+          // request. This allows revoked tokens to be rejected immediately.
+          // This also allows JWT to be shorter since some claims (notably the
+          // "scope" claim) do not need to be included in the token.
+          accessTokenMode: AccessTokenMode.stateful,
 
           getClientInfo(clientId) {
             return {
@@ -410,6 +412,10 @@ export class AppContext {
         })
       : undefined
 
+    const scopeRefGetter = entrywayAgent
+      ? new ScopeReferenceGetter(entrywayAgent, redisScratch)
+      : undefined
+
     const oauthVerifier: OAuthVerifier =
       oauthProvider ?? // OAuthProvider extends OAuthVerifier
       new OAuthVerifier({
@@ -417,6 +423,21 @@ export class AppContext {
         keyset: [await JoseKey.fromKeyLike(jwtPublicKey!, undefined, 'ES256K')],
         dpopSecret: secrets.dpopSecret,
         redis: redisScratch,
+        onDecodeToken: scopeRefGetter
+          ? async ({ payload, dpopProof }) => {
+              // @TODO drop this once oauth provider no longer accepts DPoP proof with
+              // query or fragment in "htu" claim.
+              if (dpopProof?.htu.match(/[?#]/)) {
+                oauthLogger.info(
+                  { htu: dpopProof.htu, client_id: payload.client_id },
+                  'DPoP proof "htu" contains query or fragment',
+                )
+              }
+
+              const scope = await scopeRefGetter.dereference(payload.scope)
+              return { ...payload, scope }
+            }
+          : undefined,
       })
 
     const authVerifier = new AuthVerifier(

package/src/disk-blobstore.ts

@@ -3,10 +3,16 @@ import fs from 'node:fs/promises'
 import path from 'node:path'
 import stream from 'node:stream'
 import { CID } from 'multiformats/cid'
-import { fileExists, isErrnoException, rmIfExists } from '@atproto/common'
+import {
+  aggregateErrors,
+  chunkArray,
+  fileExists,
+  isErrnoException,
+  rmIfExists,
+} from '@atproto/common'
 import { randomStr } from '@atproto/crypto'
 import { BlobNotFoundError, BlobStore } from '@atproto/repo'
-import { httpLogger as log } from './logger'
+import { blobStoreLogger as log } from './logger'
 
 export class DiskBlobStore implements BlobStore {
   constructor(
@@ -137,7 +143,18 @@ export class DiskBlobStore implements BlobStore {
   }
 
   async deleteMany(cids: CID[]): Promise<void> {
-    await Promise.all(cids.map((cid) => this.delete(cid)))
+    const errors: unknown[] = []
+    for (const chunk of chunkArray(cids, 500)) {
+      await Promise.all(
+        chunk.map((cid) =>
+          this.delete(cid).catch((err) => {
+            log.error({ err, cid: cid.toString() }, 'error deleting blob')
+            errors.push(err)
+          }),
+        ),
+      )
+    }
+    if (errors.length) throw aggregateErrors(errors)
   }
 
   async deleteAll(): Promise<void> {

package/src/lexicon/index.ts

@@ -198,6 +198,7 @@ import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscrib
 import * as ComAtprotoTempAddReservedHandle from './types/com/atproto/temp/addReservedHandle.js'
 import * as ComAtprotoTempCheckHandleAvailability from './types/com/atproto/temp/checkHandleAvailability.js'
 import * as ComAtprotoTempCheckSignupQueue from './types/com/atproto/temp/checkSignupQueue.js'
+import * as ComAtprotoTempDereferenceScope from './types/com/atproto/temp/dereferenceScope.js'
 import * as ComAtprotoTempFetchLabels from './types/com/atproto/temp/fetchLabels.js'
 import * as ComAtprotoTempRequestPhoneVerification from './types/com/atproto/temp/requestPhoneVerification.js'
 import * as ComAtprotoTempRevokeAccountCredentials from './types/com/atproto/temp/revokeAccountCredentials.js'
@@ -289,6 +290,75 @@ export const TOOLS_OZONE_MODERATION = {
   DefsTimelineEventPlcTombstone:
     'tools.ozone.moderation.defs#timelineEventPlcTombstone',
 }
+export const TOOLS_OZONE_REPORT = {
+  DefsReasonAppeal: 'tools.ozone.report.defs#reasonAppeal',
+  DefsReasonViolenceAnimalWelfare:
+    'tools.ozone.report.defs#reasonViolenceAnimalWelfare',
+  DefsReasonViolenceThreats: 'tools.ozone.report.defs#reasonViolenceThreats',
+  DefsReasonViolenceGraphicContent:
+    'tools.ozone.report.defs#reasonViolenceGraphicContent',
+  DefsReasonViolenceSelfHarm: 'tools.ozone.report.defs#reasonViolenceSelfHarm',
+  DefsReasonViolenceGlorification:
+    'tools.ozone.report.defs#reasonViolenceGlorification',
+  DefsReasonViolenceExtremistContent:
+    'tools.ozone.report.defs#reasonViolenceExtremistContent',
+  DefsReasonViolenceTrafficking:
+    'tools.ozone.report.defs#reasonViolenceTrafficking',
+  DefsReasonViolenceOther: 'tools.ozone.report.defs#reasonViolenceOther',
+  DefsReasonSexualAbuseContent:
+    'tools.ozone.report.defs#reasonSexualAbuseContent',
+  DefsReasonSexualNCII: 'tools.ozone.report.defs#reasonSexualNCII',
+  DefsReasonSexualSextortion: 'tools.ozone.report.defs#reasonSexualSextortion',
+  DefsReasonSexualDeepfake: 'tools.ozone.report.defs#reasonSexualDeepfake',
+  DefsReasonSexualAnimal: 'tools.ozone.report.defs#reasonSexualAnimal',
+  DefsReasonSexualUnlabeled: 'tools.ozone.report.defs#reasonSexualUnlabeled',
+  DefsReasonSexualOther: 'tools.ozone.report.defs#reasonSexualOther',
+  DefsReasonChildSafetyCSAM: 'tools.ozone.report.defs#reasonChildSafetyCSAM',
+  DefsReasonChildSafetyGroom: 'tools.ozone.report.defs#reasonChildSafetyGroom',
+  DefsReasonChildSafetyMinorPrivacy:
+    'tools.ozone.report.defs#reasonChildSafetyMinorPrivacy',
+  DefsReasonChildSafetyEndangerment:
+    'tools.ozone.report.defs#reasonChildSafetyEndangerment',
+  DefsReasonChildSafetyHarassment:
+    'tools.ozone.report.defs#reasonChildSafetyHarassment',
+  DefsReasonChildSafetyPromotion:
+    'tools.ozone.report.defs#reasonChildSafetyPromotion',
+  DefsReasonChildSafetyOther: 'tools.ozone.report.defs#reasonChildSafetyOther',
+  DefsReasonHarassmentTroll: 'tools.ozone.report.defs#reasonHarassmentTroll',
+  DefsReasonHarassmentTargeted:
+    'tools.ozone.report.defs#reasonHarassmentTargeted',
+  DefsReasonHarassmentHateSpeech:
+    'tools.ozone.report.defs#reasonHarassmentHateSpeech',
+  DefsReasonHarassmentDoxxing:
+    'tools.ozone.report.defs#reasonHarassmentDoxxing',
+  DefsReasonHarassmentOther: 'tools.ozone.report.defs#reasonHarassmentOther',
+  DefsReasonMisleadingBot: 'tools.ozone.report.defs#reasonMisleadingBot',
+  DefsReasonMisleadingImpersonation:
+    'tools.ozone.report.defs#reasonMisleadingImpersonation',
+  DefsReasonMisleadingSpam: 'tools.ozone.report.defs#reasonMisleadingSpam',
+  DefsReasonMisleadingScam: 'tools.ozone.report.defs#reasonMisleadingScam',
+  DefsReasonMisleadingSyntheticContent:
+    'tools.ozone.report.defs#reasonMisleadingSyntheticContent',
+  DefsReasonMisleadingMisinformation:
+    'tools.ozone.report.defs#reasonMisleadingMisinformation',
+  DefsReasonMisleadingOther: 'tools.ozone.report.defs#reasonMisleadingOther',
+  DefsReasonRuleSiteSecurity: 'tools.ozone.report.defs#reasonRuleSiteSecurity',
+  DefsReasonRuleStolenContent:
+    'tools.ozone.report.defs#reasonRuleStolenContent',
+  DefsReasonRuleProhibitedSales:
+    'tools.ozone.report.defs#reasonRuleProhibitedSales',
+  DefsReasonRuleBanEvasion: 'tools.ozone.report.defs#reasonRuleBanEvasion',
+  DefsReasonRuleOther: 'tools.ozone.report.defs#reasonRuleOther',
+  DefsReasonCivicElectoralProcess:
+    'tools.ozone.report.defs#reasonCivicElectoralProcess',
+  DefsReasonCivicDisclosure: 'tools.ozone.report.defs#reasonCivicDisclosure',
+  DefsReasonCivicInterference:
+    'tools.ozone.report.defs#reasonCivicInterference',
+  DefsReasonCivicMisinformation:
+    'tools.ozone.report.defs#reasonCivicMisinformation',
+  DefsReasonCivicImpersonation:
+    'tools.ozone.report.defs#reasonCivicImpersonation',
+}
 export const TOOLS_OZONE_TEAM = {
   DefsRoleAdmin: 'tools.ozone.team.defs#roleAdmin',
   DefsRoleModerator: 'tools.ozone.team.defs#roleModerator',
@@ -2843,6 +2913,18 @@ export class ComAtprotoTempNS {
     return this._server.xrpc.method(nsid, cfg)
   }
 
+  dereferenceScope<A extends Auth = void>(
+    cfg: MethodConfigOrHandler<
+      A,
+      ComAtprotoTempDereferenceScope.QueryParams,
+      ComAtprotoTempDereferenceScope.HandlerInput,
+      ComAtprotoTempDereferenceScope.HandlerOutput
+    >,
+  ) {
+    const nsid = 'com.atproto.temp.dereferenceScope' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
+
   fetchLabels<A extends Auth = void>(
     cfg: MethodConfigOrHandler<
       A,