@atproto/bsky 0.0.75 → 0.0.77

package/src/auth-verifier.ts

@@ -1,5 +1,6 @@
 import {
   AuthRequiredError,
+  parseReqNsid,
   verifyJwt as verifyServiceJwt,
 } from '@atproto/xrpc-server'
 import * as ui8 from 'uint8arrays'
@@ -17,6 +18,11 @@ type ReqCtx = {
   req: express.Request
 }
 
+type StandardAuthOpts = {
+  skipAudCheck?: boolean
+  lxmCheck?: (method?: string) => boolean
+}
+
 export enum RoleStatus {
   Valid,
   Invalid,
@@ -80,61 +86,55 @@ export class AuthVerifier {
   }
 
   // verifiers (arrow fns to preserve scope)
-
-  standard = async (ctx: ReqCtx): Promise<StandardOutput> => {
-    // @TODO remove! basic auth + did supported just for testing.
-    if (isBasicToken(ctx.req)) {
-      const aud = this.ownDid
-      const iss = ctx.req.headers['appview-as-did']
-      if (typeof iss !== 'string' || !iss.startsWith('did:')) {
-        throw new AuthRequiredError('bad issuer')
-      }
-      if (!this.parseRoleCreds(ctx.req).admin) {
-        throw new AuthRequiredError('bad credentials')
-      }
-      return {
-        credentials: { type: 'standard', iss, aud },
+  standardOptionalParameterized =
+    (opts: StandardAuthOpts) =>
+    async (ctx: ReqCtx): Promise<StandardOutput | NullOutput> => {
+      // @TODO remove! basic auth + did supported just for testing.
+      if (isBasicToken(ctx.req)) {
+        const aud = this.ownDid
+        const iss = ctx.req.headers['appview-as-did']
+        if (typeof iss !== 'string' || !iss.startsWith('did:')) {
+          throw new AuthRequiredError('bad issuer')
+        }
+        if (!this.parseRoleCreds(ctx.req).admin) {
+          throw new AuthRequiredError('bad credentials')
+        }
+        return {
+          credentials: { type: 'standard', iss, aud },
+        }
+      } else if (isBearerToken(ctx.req)) {
+        const { iss, aud } = await this.verifyServiceJwt(ctx, {
+          lxmCheck: opts.lxmCheck,
+          iss: null,
+          aud: null,
+        })
+        if (!opts.skipAudCheck && !this.standardAudienceDids.has(aud)) {
+          throw new AuthRequiredError(
+            'jwt audience does not match service did',
+            'BadJwtAudience',
+          )
+        }
+        return {
+          credentials: {
+            type: 'standard',
+            iss,
+            aud,
+          },
+        }
+      } else {
+        return this.nullCreds()
       }
     }
-    const { iss, aud } = await this.verifyServiceJwt(ctx, {
-      aud: null,
-      iss: null,
-    })
-    if (!this.standardAudienceDids.has(aud)) {
-      throw new AuthRequiredError(
-        'jwt audience does not match service did',
-        'BadJwtAudience',
-      )
-    }
-    return {
-      credentials: {
-        type: 'standard',
-        iss,
-        aud,
-      },
-    }
-  }
 
-  standardOptional = async (
-    ctx: ReqCtx,
-  ): Promise<StandardOutput | NullOutput> => {
-    if (isBearerToken(ctx.req) || isBasicToken(ctx.req)) {
-      return this.standard(ctx)
-    }
-    return this.nullCreds()
-  }
+  standardOptional: (ctx: ReqCtx) => Promise<StandardOutput | NullOutput> =
+    this.standardOptionalParameterized({})
 
-  standardOptionalAnyAud = async (
-    ctx: ReqCtx,
-  ): Promise<StandardOutput | NullOutput> => {
-    if (!isBearerToken(ctx.req)) {
-      return this.nullCreds()
+  standard = async (ctx: ReqCtx): Promise<StandardOutput> => {
+    const output = await this.standardOptional(ctx)
+    if (output.credentials.type === 'none') {
+      throw new AuthRequiredError(undefined, 'AuthMissing')
     }
-    const { iss, aud } = await this.verifyServiceJwt(ctx, {
-      aud: null,
-      iss: null,
-    })
-    return { credentials: { type: 'standard', iss, aud } }
+    return output as StandardOutput
   }
 
   role = (ctx: ReqCtx): RoleOutput => {
@@ -215,7 +215,11 @@ export class AuthVerifier {
 
   async verifyServiceJwt(
     reqCtx: ReqCtx,
-    opts: { aud: string | null; iss: string[] | null },
+    opts: {
+      iss: string[] | null
+      aud: string | null
+      lxmCheck?: (method?: string) => boolean
+    },
   ) {
     const getSigningKey = async (
       iss: string,
@@ -243,17 +247,40 @@ export class AuthVerifier {
       }
       return didKey
     }
+    const assertLxmCheck = () => {
+      const lxm = parseReqNsid(reqCtx.req)
+      if (
+        (opts.lxmCheck && !opts.lxmCheck(payload.lxm)) ||
+        (!opts.lxmCheck && payload.lxm !== lxm)
+      ) {
+        throw new AuthRequiredError(
+          payload.lxm !== undefined
+            ? `bad jwt lexicon method ("lxm"). must match: ${lxm}`
+            : `missing jwt lexicon method ("lxm"). must match: ${lxm}`,
+          'BadJwtLexiconMethod',
+        )
+      }
+    }
 
     const jwtStr = bearerTokenFromReq(reqCtx.req)
     if (!jwtStr) {
       throw new AuthRequiredError('missing jwt', 'MissingJwt')
     }
+    // if validating additional scopes, skip scope check in initial validation & follow up afterwards
     const payload = await verifyServiceJwt(
       jwtStr,
       opts.aud,
       null,
       getSigningKey,
     )
+    if (
+      !payload.iss.endsWith('#atproto_labeler') ||
+      payload.lxm !== undefined
+    ) {
+      // @TODO currently permissive of labelers who dont set lxm yet.
+      // we'll allow ozone self-hosters to upgrade before removing this condition.
+      assertLxmCheck()
+    }
     return { iss: payload.iss, aud: payload.aud }
   }
 

package/src/data-plane/server/db/database-schema.ts

@@ -6,7 +6,8 @@ import * as post from './tables/post'
 import * as postEmbed from './tables/post-embed'
 import * as postAgg from './tables/post-agg'
 import * as repost from './tables/repost'
-import * as threadGate from './tables/thread-gate'
+import * as threadgate from './tables/thread-gate'
+import * as postgate from './tables/post-gate'
 import * as feedItem from './tables/feed-item'
 import * as follow from './tables/follow'
 import * as like from './tables/like'
@@ -35,6 +36,7 @@ import * as taggedSuggestion from './tables/tagged-suggestion'
 import * as blobTakedown from './tables/blob-takedown'
 import * as labeler from './tables/labeler'
 import * as starterPack from './tables/starter-pack'
+import * as quote from './tables/quote'
 
 export type DatabaseSchemaType = duplicateRecord.PartialDB &
   profile.PartialDB &
@@ -43,7 +45,8 @@ export type DatabaseSchemaType = duplicateRecord.PartialDB &
   postEmbed.PartialDB &
   postAgg.PartialDB &
   repost.PartialDB &
-  threadGate.PartialDB &
+  threadgate.PartialDB &
+  postgate.PartialDB &
   feedItem.PartialDB &
   follow.PartialDB &
   like.PartialDB &
@@ -71,7 +74,8 @@ export type DatabaseSchemaType = duplicateRecord.PartialDB &
   blobTakedown.PartialDB &
   labeler.PartialDB &
   starterPack.PartialDB &
-  taggedSuggestion.PartialDB
+  taggedSuggestion.PartialDB &
+  quote.PartialDB
 
 export type DatabaseSchema = Kysely<DatabaseSchemaType>
 

package/src/data-plane/server/db/migrations/20240723T220700077Z-quotes-post-aggs.ts

@@ -0,0 +1,12 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('post_agg')
+    .addColumn('quoteCount', 'bigint', (col) => col.notNull().defaultTo(0))
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post_agg').dropColumn('quoteCount').execute()
+}

package/src/data-plane/server/db/migrations/20240723T220703655Z-quotes.ts

@@ -0,0 +1,28 @@
+import { Kysely, sql } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('quote')
+    .addColumn('uri', 'varchar', (col) => col.primaryKey())
+    .addColumn('cid', 'varchar', (col) => col.notNull())
+    .addColumn('subject', 'varchar', (col) => col.notNull())
+    .addColumn('subjectCid', 'varchar', (col) => col.notNull())
+    .addColumn('createdAt', 'varchar', (col) => col.notNull())
+    .addColumn('indexedAt', 'varchar', (col) => col.notNull())
+    .addColumn('sortAt', 'varchar', (col) =>
+      col
+        .generatedAlwaysAs(sql`least("createdAt", "indexedAt")`)
+        .stored()
+        .notNull(),
+    )
+    .execute()
+  await db.schema
+    .createIndex('quote_subject_cursor_idx')
+    .on('quote')
+    .columns(['subject', 'sortAt', 'cid'])
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('quote').execute()
+}

package/src/data-plane/server/db/migrations/20240801T193939827Z-post-gate.ts

@@ -0,0 +1,17 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('post_gate')
+    .addColumn('uri', 'varchar', (col) => col.primaryKey())
+    .addColumn('cid', 'varchar', (col) => col.notNull())
+    .addColumn('creator', 'varchar', (col) => col.notNull())
+    .addColumn('postUri', 'varchar', (col) => col.notNull().unique())
+    .addColumn('createdAt', 'varchar', (col) => col.notNull())
+    .addColumn('indexedAt', 'varchar', (col) => col.notNull())
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('post_gate').execute()
+}

package/src/data-plane/server/db/migrations/20240808T224251220Z-post-gate-flags.ts

@@ -0,0 +1,25 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('post')
+    .addColumn('violatesEmbeddingRules', 'boolean')
+    .execute()
+  await db.schema
+    .alterTable('post')
+    .addColumn('hasThreadGate', 'boolean')
+    .execute()
+  await db.schema
+    .alterTable('post')
+    .addColumn('hasPostGate', 'boolean')
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .alterTable('post')
+    .dropColumn('violatesEmbeddingRules')
+    .execute()
+  await db.schema.alterTable('post').dropColumn('hasThreadGate').execute()
+  await db.schema.alterTable('post').dropColumn('hasPostGate').execute()
+}

package/src/data-plane/server/db/migrations/index.ts

@@ -38,3 +38,7 @@ export * as _20240530T170337073Z from './20240530T170337073Z-account-deactivatio
 export * as _20240606T171229898Z from './20240606T171229898Z-thread-mutes'
 export * as _20240606T222548219Z from './20240606T222548219Z-starter-packs'
 export * as _20240719T203853939Z from './20240719T203853939Z-priority-notifs'
+export * as _20240723T220700077Z from './20240723T220700077Z-quotes-post-aggs'
+export * as _20240723T220703655Z from './20240723T220703655Z-quotes'
+export * as _20240801T193939827Z from './20240801T193939827Z-post-gate'
+export * as _20240808T224251220Z from './20240808T224251220Z-post-gate-flags'

package/src/data-plane/server/db/tables/post-agg.ts

@@ -7,6 +7,7 @@ export interface PostAgg {
   likeCount: Generated<number>
   replyCount: Generated<number>
   repostCount: Generated<number>
+  quoteCount: Generated<number>
 }
 
 export type PartialDB = {

package/src/data-plane/server/db/tables/post-gate.ts

@@ -0,0 +1,12 @@
+const tableName = 'post_gate'
+
+export interface Postgate {
+  uri: string
+  cid: string
+  creator: string
+  postUri: string
+  createdAt: string
+  indexedAt: string
+}
+
+export type PartialDB = { [tableName]: Postgate }

package/src/data-plane/server/db/tables/post.ts

@@ -15,6 +15,9 @@ export interface Post {
   tags: string[] | null
   invalidReplyRoot: boolean | null
   violatesThreadGate: boolean | null
+  violatesEmbeddingRules: boolean | null
+  hasThreadGate: boolean | null
+  hasPostGate: boolean | null
   createdAt: string
   indexedAt: string
   sortAt: GeneratedAlways<string>

package/src/data-plane/server/db/tables/quote.ts

@@ -0,0 +1,15 @@
+import { GeneratedAlways } from 'kysely'
+
+const tableName = 'quote'
+
+export interface Quote {
+  uri: string
+  cid: string
+  subject: string
+  subjectCid: string
+  createdAt: string
+  indexedAt: string
+  sortAt: GeneratedAlways<string>
+}
+
+export type PartialDB = { [tableName]: Quote }

package/src/data-plane/server/indexing/index.ts

@@ -17,6 +17,7 @@ import { Database } from '../db'
 import { Actor } from '../db/tables/actor'
 import * as Post from './plugins/post'
 import * as Threadgate from './plugins/thread-gate'
+import * as Postgate from './plugins/post-gate'
 import * as Like from './plugins/like'
 import * as Repost from './plugins/repost'
 import * as Follow from './plugins/follow'
@@ -38,6 +39,7 @@ export class IndexingService {
   records: {
     post: Post.PluginType
     threadGate: Threadgate.PluginType
+    postGate: Postgate.PluginType
     like: Like.PluginType
     repost: Repost.PluginType
     follow: Follow.PluginType
@@ -60,6 +62,7 @@ export class IndexingService {
     this.records = {
       post: Post.makePlugin(this.db, this.background),
       threadGate: Threadgate.makePlugin(this.db, this.background),
+      postGate: Postgate.makePlugin(this.db, this.background),
       like: Like.makePlugin(this.db, this.background),
       repost: Repost.makePlugin(this.db, this.background),
       follow: Follow.makePlugin(this.db, this.background),
@@ -365,6 +368,10 @@ export class IndexingService {
       .deleteFrom('thread_gate')
       .where('creator', '=', did)
       .execute()
+    await this.db.db
+      .deleteFrom('post_gate')
+      .where('creator', '=', did)
+      .execute()
     // notifications
     await this.db.db
       .deleteFrom('notification')

package/src/data-plane/server/indexing/plugins/post-gate.ts

@@ -0,0 +1,104 @@
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
+import { InvalidRequestError } from '@atproto/xrpc-server'
+import { CID } from 'multiformats/cid'
+import * as Postgate from '../../../../lexicon/types/app/bsky/feed/postgate'
+import * as lex from '../../../../lexicon/lexicons'
+import { DatabaseSchema, DatabaseSchemaType } from '../../db/database-schema'
+import { Database } from '../../db'
+import RecordProcessor from '../processor'
+import { BackgroundQueue } from '../../background'
+
+const lexId = lex.ids.AppBskyFeedPostgate
+type IndexedGate = DatabaseSchemaType['post_gate']
+
+const insertFn = async (
+  db: DatabaseSchema,
+  uri: AtUri,
+  cid: CID,
+  obj: Postgate.Record,
+  timestamp: string,
+): Promise<IndexedGate | null> => {
+  const postUri = new AtUri(obj.post)
+  if (postUri.host !== uri.host || postUri.rkey !== uri.rkey) {
+    throw new InvalidRequestError(
+      'Creator and rkey of post gate does not match its post',
+    )
+  }
+  const inserted = await db
+    .insertInto('post_gate')
+    .values({
+      uri: uri.toString(),
+      cid: cid.toString(),
+      creator: uri.host,
+      postUri: obj.post,
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
+      indexedAt: timestamp,
+    })
+    .onConflict((oc) => oc.doNothing())
+    .returningAll()
+    .executeTakeFirst()
+  await db
+    .updateTable('post')
+    .where('uri', '=', postUri.toString())
+    .set({ hasPostGate: true })
+    .executeTakeFirst()
+  return inserted || null
+}
+
+const findDuplicate = async (
+  db: DatabaseSchema,
+  _uri: AtUri,
+  obj: Postgate.Record,
+): Promise<AtUri | null> => {
+  const found = await db
+    .selectFrom('post_gate')
+    .where('postUri', '=', obj.post)
+    .selectAll()
+    .executeTakeFirst()
+  return found ? new AtUri(found.uri) : null
+}
+
+const notifsForInsert = () => {
+  return []
+}
+
+const deleteFn = async (
+  db: DatabaseSchema,
+  uri: AtUri,
+): Promise<IndexedGate | null> => {
+  const deleted = await db
+    .deleteFrom('post_gate')
+    .where('uri', '=', uri.toString())
+    .returningAll()
+    .executeTakeFirst()
+  if (deleted) {
+    await db
+      .updateTable('post')
+      .where('uri', '=', deleted.postUri)
+      .set({ hasPostGate: false })
+      .executeTakeFirst()
+  }
+  return deleted || null
+}
+
+const notifsForDelete = () => {
+  return { notifs: [], toDelete: [] }
+}
+
+export type PluginType = RecordProcessor<Postgate.Record, IndexedGate>
+
+export const makePlugin = (
+  db: Database,
+  background: BackgroundQueue,
+): PluginType => {
+  return new RecordProcessor(db, background, {
+    lexId,
+    insertFn,
+    findDuplicate,
+    deleteFn,
+    notifsForInsert,
+    notifsForDelete,
+  })
+}
+
+export default makePlugin