@atproto/bsky 0.0.25 → 0.0.26

package/src/config.ts

@@ -31,6 +31,16 @@ export interface ServerConfigValues {
   imgUriEndpoint?: string
   blobCacheLocation?: string
   searchEndpoint?: string
+  bsyncUrl?: string
+  bsyncApiKey?: string
+  bsyncHttpVersion?: '1.1' | '2'
+  bsyncIgnoreBadTls?: boolean
+  bsyncOnlyMutes?: boolean
+  courierUrl?: string
+  courierApiKey?: string
+  courierHttpVersion?: '1.1' | '2'
+  courierIgnoreBadTls?: boolean
+  courierOnlyRegistration?: boolean
   adminPassword: string
   moderatorPassword: string
   triagePassword: string
@@ -88,6 +98,25 @@ export class ServerConfig {
     const imgUriEndpoint = process.env.IMG_URI_ENDPOINT
     const blobCacheLocation = process.env.BLOB_CACHE_LOC
     const searchEndpoint = process.env.SEARCH_ENDPOINT
+    const bsyncUrl = process.env.BSKY_BSYNC_URL || undefined
+    const bsyncApiKey = process.env.BSKY_BSYNC_API_KEY || undefined
+    const bsyncHttpVersion = process.env.BSKY_BSYNC_HTTP_VERSION || '2'
+    const bsyncIgnoreBadTls = process.env.BSKY_BSYNC_IGNORE_BAD_TLS === 'true'
+    const bsyncOnlyMutes = process.env.BSKY_BSYNC_ONLY_MUTES === 'true'
+    assert(!bsyncOnlyMutes || bsyncUrl, 'bsync-only mutes requires a bsync url')
+    assert(bsyncHttpVersion === '1.1' || bsyncHttpVersion === '2')
+    const courierUrl = process.env.BSKY_COURIER_URL || undefined
+    const courierApiKey = process.env.BSKY_COURIER_API_KEY || undefined
+    const courierHttpVersion = process.env.BSKY_COURIER_HTTP_VERSION || '2'
+    const courierIgnoreBadTls =
+      process.env.BSKY_COURIER_IGNORE_BAD_TLS === 'true'
+    const courierOnlyRegistration =
+      process.env.BSKY_COURIER_ONLY_REGISTRATION === 'true'
+    assert(
+      !courierOnlyRegistration || courierUrl,
+      'courier-only registration requires a courier url',
+    )
+    assert(courierHttpVersion === '1.1' || courierHttpVersion === '2')
     const dbPrimaryPostgresUrl =
       overrides?.dbPrimaryPostgresUrl || process.env.DB_PRIMARY_POSTGRES_URL
     let dbReplicaPostgresUrls = overrides?.dbReplicaPostgresUrls
@@ -152,6 +181,16 @@ export class ServerConfig {
       imgUriEndpoint,
       blobCacheLocation,
       searchEndpoint,
+      bsyncUrl,
+      bsyncApiKey,
+      bsyncHttpVersion,
+      bsyncIgnoreBadTls,
+      bsyncOnlyMutes,
+      courierUrl,
+      courierApiKey,
+      courierHttpVersion,
+      courierIgnoreBadTls,
+      courierOnlyRegistration,
       adminPassword,
       moderatorPassword,
       triagePassword,
@@ -268,6 +307,46 @@ export class ServerConfig {
     return this.cfg.searchEndpoint
   }
 
+  get bsyncUrl() {
+    return this.cfg.bsyncUrl
+  }
+
+  get bsyncApiKey() {
+    return this.cfg.bsyncApiKey
+  }
+
+  get bsyncOnlyMutes() {
+    return this.cfg.bsyncOnlyMutes
+  }
+
+  get bsyncHttpVersion() {
+    return this.cfg.bsyncHttpVersion
+  }
+
+  get bsyncIgnoreBadTls() {
+    return this.cfg.bsyncIgnoreBadTls
+  }
+
+  get courierUrl() {
+    return this.cfg.courierUrl
+  }
+
+  get courierApiKey() {
+    return this.cfg.courierApiKey
+  }
+
+  get courierHttpVersion() {
+    return this.cfg.courierHttpVersion
+  }
+
+  get courierIgnoreBadTls() {
+    return this.cfg.courierIgnoreBadTls
+  }
+
+  get courierOnlyRegistration() {
+    return this.cfg.courierOnlyRegistration
+  }
+
   get adminPassword() {
     return this.cfg.adminPassword
   }

package/src/context.ts

@@ -10,9 +10,10 @@ import { Services } from './services'
 import DidRedisCache from './did-cache'
 import { BackgroundQueue } from './background'
 import { MountedAlgos } from './feed-gen/types'
-import { NotificationServer } from './notifications'
 import { Redis } from './redis'
 import { AuthVerifier } from './auth-verifier'
+import { BsyncClient } from './bsync'
+import { CourierClient } from './courier'
 
 export class AppContext {
   constructor(
@@ -27,8 +28,9 @@ export class AppContext {
       redis: Redis
       backgroundQueue: BackgroundQueue
       searchAgent?: AtpAgent
+      bsyncClient?: BsyncClient
+      courierClient?: CourierClient
       algos: MountedAlgos
-      notifServer: NotificationServer
       authVerifier: AuthVerifier
     },
   ) {}
@@ -69,14 +71,18 @@ export class AppContext {
     return this.opts.redis
   }
 
-  get notifServer(): NotificationServer {
-    return this.opts.notifServer
-  }
-
   get searchAgent(): AtpAgent | undefined {
     return this.opts.searchAgent
   }
 
+  get bsyncClient(): BsyncClient | undefined {
+    return this.opts.bsyncClient
+  }
+
+  get courierClient(): CourierClient | undefined {
+    return this.opts.courierClient
+  }
+
   get authVerifier(): AuthVerifier {
     return this.opts.authVerifier
   }

package/src/courier.ts

@@ -0,0 +1,41 @@
+import { Service } from './proto/courier_connect'
+import {
+  Code,
+  ConnectError,
+  PromiseClient,
+  createPromiseClient,
+  Interceptor,
+} from '@connectrpc/connect'
+import {
+  createConnectTransport,
+  ConnectTransportOptions,
+} from '@connectrpc/connect-node'
+
+export type CourierClient = PromiseClient<typeof Service>
+
+export const createCourierClient = (
+  opts: ConnectTransportOptions,
+): CourierClient => {
+  const transport = createConnectTransport(opts)
+  return createPromiseClient(Service, transport)
+}
+
+export { Code }
+
+export const isCourierError = (
+  err: unknown,
+  code?: Code,
+): err is ConnectError => {
+  if (err instanceof ConnectError) {
+    return !code || err.code === code
+  }
+  return false
+}
+
+export const authWithApiKey =
+  (apiKey: string): Interceptor =>
+  (next) =>
+  (req) => {
+    req.header.set('authorization', `Bearer ${apiKey}`)
+    return next(req)
+  }

package/src/db/database-schema.ts

@@ -30,6 +30,7 @@ import * as algo from './tables/algo'
 import * as viewParam from './tables/view-param'
 import * as suggestedFollow from './tables/suggested-follow'
 import * as suggestedFeed from './tables/suggested-feed'
+import * as taggedSuggestion from './tables/tagged-suggestion'
 import * as blobTakedown from './tables/blob-takedown'
 
 export type DatabaseSchemaType = duplicateRecord.PartialDB &
@@ -63,6 +64,7 @@ export type DatabaseSchemaType = duplicateRecord.PartialDB &
   viewParam.PartialDB &
   suggestedFollow.PartialDB &
   suggestedFeed.PartialDB &
+  taggedSuggestion.PartialDB &
   blobTakedown.PartialDB
 
 export type DatabaseSchema = Kysely<DatabaseSchemaType>

package/src/db/migrations/20240124T023719200Z-tagged-suggestions.ts

@@ -0,0 +1,15 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('tagged_suggestion')
+    .addColumn('tag', 'varchar', (col) => col.notNull())
+    .addColumn('subject', 'varchar', (col) => col.notNull())
+    .addColumn('subjectType', 'varchar', (col) => col.notNull())
+    .addPrimaryKeyConstraint('tagged_suggestion_pkey', ['tag', 'subject'])
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('tagged_suggestion').execute()
+}

package/src/db/migrations/index.ts

@@ -33,3 +33,4 @@ export * as _20230929T192920807Z from './20230929T192920807Z-record-cursor-index
 export * as _20231003T202833377Z from './20231003T202833377Z-create-moderation-subject-status'
 export * as _20231205T000257238Z from './20231205T000257238Z-remove-did-cache'
 export * as _20231220T225126090Z from './20231220T225126090Z-blob-takedowns'
+export * as _20240124T023719200Z from './20240124T023719200Z-tagged-suggestions'

package/src/db/tables/tagged-suggestion.ts

@@ -0,0 +1,11 @@
+export const tableName = 'tagged_suggestion'
+
+export interface TaggedSuggestion {
+  tag: string
+  subject: string
+  subjectType: string
+}
+
+export type PartialDB = {
+  [tableName]: TaggedSuggestion
+}

package/src/index.ts

@@ -29,11 +29,12 @@ import {
 } from './image/invalidator'
 import { BackgroundQueue } from './background'
 import { MountedAlgos } from './feed-gen/types'
-import { NotificationServer } from './notifications'
 import { AtpAgent } from '@atproto/api'
 import { Keypair } from '@atproto/crypto'
 import { Redis } from './redis'
 import { AuthVerifier } from './auth-verifier'
+import { authWithApiKey as bsyncAuth, createBsyncClient } from './bsync'
+import { authWithApiKey as courierAuth, createCourierClient } from './courier'
 
 export type { ServerConfigValues } from './config'
 export type { MountedAlgos } from './feed-gen/types'
@@ -112,7 +113,6 @@ export class BskyAppView {
 
     const backgroundQueue = new BackgroundQueue(db.getPrimary())
 
-    const notifServer = new NotificationServer(db.getPrimary())
     const searchAgent = config.searchEndpoint
       ? new AtpAgent({ service: config.searchEndpoint })
       : undefined
@@ -135,6 +135,28 @@ export class BskyAppView {
       triagePass: config.triagePassword,
     })
 
+    const bsyncClient = config.bsyncUrl
+      ? createBsyncClient({
+          baseUrl: config.bsyncUrl,
+          httpVersion: config.bsyncHttpVersion ?? '2',
+          nodeOptions: { rejectUnauthorized: !config.bsyncIgnoreBadTls },
+          interceptors: config.bsyncApiKey
+            ? [bsyncAuth(config.bsyncApiKey)]
+            : [],
+        })
+      : undefined
+
+    const courierClient = config.courierUrl
+      ? createCourierClient({
+          baseUrl: config.courierUrl,
+          httpVersion: config.courierHttpVersion ?? '2',
+          nodeOptions: { rejectUnauthorized: !config.courierIgnoreBadTls },
+          interceptors: config.courierApiKey
+            ? [courierAuth(config.courierApiKey)]
+            : [],
+        })
+      : undefined
+
     const ctx = new AppContext({
       db,
       cfg: config,
@@ -146,8 +168,9 @@ export class BskyAppView {
       redis,
       backgroundQueue,
       searchAgent,
+      bsyncClient,
+      courierClient,
       algos,
-      notifServer,
       authVerifier,
     })
 

package/src/indexer/config.ts

@@ -22,6 +22,10 @@ export interface IndexerConfigValues {
   fuzzyFalsePositiveB64?: string
   labelerKeywords: Record<string, string>
   moderationPushUrl: string
+  courierUrl?: string
+  courierApiKey?: string
+  courierHttpVersion?: '1.1' | '2'
+  courierIgnoreBadTls?: boolean
   indexerConcurrency?: number
   indexerPartitionIds: number[]
   indexerPartitionBatchSize?: number
@@ -72,6 +76,18 @@ export class IndexerConfig {
       process.env.MODERATION_PUSH_URL ||
       undefined
     assert(moderationPushUrl)
+    const courierUrl =
+      overrides?.courierUrl || process.env.BSKY_COURIER_URL || undefined
+    const courierApiKey =
+      overrides?.courierApiKey || process.env.BSKY_COURIER_API_KEY || undefined
+    const courierHttpVersion =
+      overrides?.courierHttpVersion ||
+      process.env.BSKY_COURIER_HTTP_VERSION ||
+      '2'
+    const courierIgnoreBadTls =
+      overrides?.courierIgnoreBadTls ||
+      process.env.BSKY_COURIER_IGNORE_BAD_TLS === 'true'
+    assert(courierHttpVersion === '1.1' || courierHttpVersion === '2')
     const hiveApiKey = process.env.HIVE_API_KEY || undefined
     const abyssEndpoint = process.env.ABYSS_ENDPOINT
     const abyssPassword = process.env.ABYSS_PASSWORD
@@ -114,6 +130,10 @@ export class IndexerConfig {
       didCacheMaxTTL,
       handleResolveNameservers,
       moderationPushUrl,
+      courierUrl,
+      courierApiKey,
+      courierHttpVersion,
+      courierIgnoreBadTls,
       hiveApiKey,
       abyssEndpoint,
       abyssPassword,
@@ -185,6 +205,22 @@ export class IndexerConfig {
     return this.cfg.moderationPushUrl
   }
 
+  get courierUrl() {
+    return this.cfg.courierUrl
+  }
+
+  get courierApiKey() {
+    return this.cfg.courierApiKey
+  }
+
+  get courierHttpVersion() {
+    return this.cfg.courierHttpVersion
+  }
+
+  get courierIgnoreBadTls() {
+    return this.cfg.courierIgnoreBadTls
+  }
+
   get hiveApiKey() {
     return this.cfg.hiveApiKey
   }

package/src/indexer/context.ts

@@ -6,6 +6,7 @@ import { BackgroundQueue } from '../background'
 import DidSqlCache from '../did-cache'
 import { Redis } from '../redis'
 import { AutoModerator } from '../auto-moderator'
+import { NotificationServer } from '../notifications'
 
 export class IndexerContext {
   constructor(
@@ -19,6 +20,7 @@ export class IndexerContext {
       didCache: DidSqlCache
       backgroundQueue: BackgroundQueue
       autoMod: AutoModerator
+      notifServer?: NotificationServer
     },
   ) {}
 
@@ -57,6 +59,10 @@ export class IndexerContext {
   get autoMod(): AutoModerator {
     return this.opts.autoMod
   }
+
+  get notifServer(): NotificationServer | undefined {
+    return this.opts.notifServer
+  }
 }
 
 export default IndexerContext

package/src/indexer/index.ts

@@ -11,8 +11,13 @@ import { createServices } from './services'
 import { IndexerSubscription } from './subscription'
 import { AutoModerator } from '../auto-moderator'
 import { Redis } from '../redis'
-import { NotificationServer } from '../notifications'
+import {
+  CourierNotificationServer,
+  GorushNotificationServer,
+  NotificationServer,
+} from '../notifications'
 import { CloseFn, createServer, startServer } from './server'
+import { authWithApiKey as courierAuth, createCourierClient } from '../courier'
 
 export { IndexerConfig } from './config'
 export type { IndexerConfigValues } from './config'
@@ -60,9 +65,27 @@ export class BskyIndexer {
       backgroundQueue,
     })
 
-    const notifServer = cfg.pushNotificationEndpoint
-      ? new NotificationServer(db, cfg.pushNotificationEndpoint)
+    const courierClient = cfg.courierUrl
+      ? createCourierClient({
+          baseUrl: cfg.courierUrl,
+          httpVersion: cfg.courierHttpVersion ?? '2',
+          nodeOptions: { rejectUnauthorized: !cfg.courierIgnoreBadTls },
+          interceptors: cfg.courierApiKey
+            ? [courierAuth(cfg.courierApiKey)]
+            : [],
+        })
       : undefined
+
+    let notifServer: NotificationServer | undefined
+    if (courierClient) {
+      notifServer = new CourierNotificationServer(db, courierClient)
+    } else if (cfg.pushNotificationEndpoint) {
+      notifServer = new GorushNotificationServer(
+        db,
+        cfg.pushNotificationEndpoint,
+      )
+    }
+
     const services = createServices({
       idResolver,
       autoMod,
@@ -79,6 +102,7 @@ export class BskyIndexer {
       didCache,
       backgroundQueue,
       autoMod,
+      notifServer,
     })
     const sub = new IndexerSubscription(ctx, {
       partitionIds: cfg.indexerPartitionIds,

package/src/ingester/config.ts

@@ -10,6 +10,10 @@ export interface IngesterConfigValues {
   redisPassword?: string
   repoProvider: string
   labelProvider?: string
+  bsyncUrl?: string
+  bsyncApiKey?: string
+  bsyncHttpVersion?: '1.1' | '2'
+  bsyncIgnoreBadTls?: boolean
   ingesterPartitionCount: number
   ingesterNamespace?: string
   ingesterSubLockId?: number
@@ -42,6 +46,16 @@ export class IngesterConfig {
       overrides?.redisPassword || process.env.REDIS_PASSWORD || undefined
     const repoProvider = overrides?.repoProvider || process.env.REPO_PROVIDER // E.g. ws://abc.com:4000
     const labelProvider = overrides?.labelProvider || process.env.LABEL_PROVIDER
+    const bsyncUrl =
+      overrides?.bsyncUrl || process.env.BSKY_BSYNC_URL || undefined
+    const bsyncApiKey =
+      overrides?.bsyncApiKey || process.env.BSKY_BSYNC_API_KEY || undefined
+    const bsyncHttpVersion =
+      overrides?.bsyncHttpVersion || process.env.BSKY_BSYNC_HTTP_VERSION || '2'
+    const bsyncIgnoreBadTls =
+      overrides?.bsyncIgnoreBadTls ||
+      process.env.BSKY_BSYNC_IGNORE_BAD_TLS === 'true'
+    assert(bsyncHttpVersion === '1.1' || bsyncHttpVersion === '2')
     const ingesterPartitionCount =
       overrides?.ingesterPartitionCount ||
       maybeParseInt(process.env.INGESTER_PARTITION_COUNT)
@@ -72,6 +86,10 @@ export class IngesterConfig {
       redisPassword,
       repoProvider,
       labelProvider,
+      bsyncUrl,
+      bsyncApiKey,
+      bsyncHttpVersion,
+      bsyncIgnoreBadTls,
       ingesterPartitionCount,
       ingesterSubLockId,
       ingesterNamespace,
@@ -117,6 +135,22 @@ export class IngesterConfig {
     return this.cfg.labelProvider
   }
 
+  get bsyncUrl() {
+    return this.cfg.bsyncUrl
+  }
+
+  get bsyncApiKey() {
+    return this.cfg.bsyncApiKey
+  }
+
+  get bsyncHttpVersion() {
+    return this.cfg.bsyncHttpVersion
+  }
+
+  get bsyncIgnoreBadTls() {
+    return this.cfg.bsyncIgnoreBadTls
+  }
+
   get ingesterPartitionCount() {
     return this.cfg.ingesterPartitionCount
   }

package/src/ingester/context.ts

@@ -2,6 +2,7 @@ import { PrimaryDatabase } from '../db'
 import { Redis } from '../redis'
 import { IngesterConfig } from './config'
 import { LabelSubscription } from './label-subscription'
+import { MuteSubscription } from './mute-subscription'
 
 export class IngesterContext {
   constructor(
@@ -10,6 +11,7 @@ export class IngesterContext {
       redis: Redis
       cfg: IngesterConfig
       labelSubscription?: LabelSubscription
+      muteSubscription?: MuteSubscription
     },
   ) {}
 
@@ -28,6 +30,10 @@ export class IngesterContext {
   get labelSubscription(): LabelSubscription | undefined {
     return this.opts.labelSubscription
   }
+
+  get muteSubscription(): MuteSubscription | undefined {
+    return this.opts.muteSubscription
+  }
 }
 
 export default IngesterContext

package/src/ingester/index.ts

@@ -5,7 +5,9 @@ import { Redis } from '../redis'
 import { IngesterConfig } from './config'
 import { IngesterContext } from './context'
 import { IngesterSubscription } from './subscription'
+import { authWithApiKey, createBsyncClient } from '../bsync'
 import { LabelSubscription } from './label-subscription'
+import { MuteSubscription } from './mute-subscription'
 
 export { IngesterConfig } from './config'
 export type { IngesterConfigValues } from './config'
@@ -27,14 +29,28 @@ export class BskyIngester {
     cfg: IngesterConfig
   }): BskyIngester {
     const { db, redis, cfg } = opts
+    const bsyncClient = cfg.bsyncUrl
+      ? createBsyncClient({
+          baseUrl: cfg.bsyncUrl,
+          httpVersion: cfg.bsyncHttpVersion ?? '2',
+          nodeOptions: { rejectUnauthorized: !cfg.bsyncIgnoreBadTls },
+          interceptors: cfg.bsyncApiKey
+            ? [authWithApiKey(cfg.bsyncApiKey)]
+            : [],
+        })
+      : undefined
     const labelSubscription = cfg.labelProvider
       ? new LabelSubscription(db, cfg.labelProvider)
       : undefined
+    const muteSubscription = bsyncClient
+      ? new MuteSubscription(db, redis, bsyncClient)
+      : undefined
     const ctx = new IngesterContext({
       db,
       redis,
       cfg,
       labelSubscription,
+      muteSubscription,
     })
     const sub = new IngesterSubscription(ctx, {
       service: cfg.repoProvider,
@@ -73,11 +89,13 @@ export class BskyIngester {
       )
     }, 500)
     await this.ctx.labelSubscription?.start()
+    await this.ctx.muteSubscription?.start()
     this.sub.run()
     return this
   }
 
   async destroy(opts?: { skipDb: boolean }): Promise<void> {
+    await this.ctx.muteSubscription?.destroy()
     await this.ctx.labelSubscription?.destroy()
     await this.sub.destroy()
     clearInterval(this.subStatsInterval)