@atproto/bsync 0.0.0

package/dist/logger.d.ts

@@ -0,0 +1,4 @@
+import { subsystemLogger } from '@atproto/common';
+export declare const dbLogger: ReturnType<typeof subsystemLogger>;
+export declare const httpLogger: ReturnType<typeof subsystemLogger>;
+export declare const loggerMiddleware: import("pino-http").HttpLogger;

package/dist/routes/add-mute-operation.d.ts

@@ -0,0 +1,5 @@
+import { ServiceImpl } from '@connectrpc/connect';
+import { Service } from '../gen/bsync_connect';
+import AppContext from '../context';
+declare const _default: (ctx: AppContext) => Partial<ServiceImpl<typeof Service>>;
+export default _default;

package/dist/routes/auth.d.ts

@@ -0,0 +1,3 @@
+import { HandlerContext } from '@connectrpc/connect';
+import AppContext from '../context';
+export declare const authWithApiKey: (ctx: AppContext, handlerCtx: HandlerContext) => void;

package/dist/routes/index.d.ts

@@ -0,0 +1,4 @@
+import { ConnectRouter } from '@connectrpc/connect';
+import AppContext from '../context';
+declare const _default: (ctx: AppContext) => (router: ConnectRouter) => ConnectRouter;
+export default _default;

package/dist/routes/scan-mute-operations.d.ts

@@ -0,0 +1,5 @@
+import { ServiceImpl } from '@connectrpc/connect';
+import { Service } from '../gen/bsync_connect';
+import AppContext from '../context';
+declare const _default: (ctx: AppContext) => Partial<ServiceImpl<typeof Service>>;
+export default _default;

package/jest.config.js

@@ -0,0 +1,6 @@
+const base = require('../../jest.config.base.js')
+
+module.exports = {
+  ...base,
+  displayName: 'Bsync',
+}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@atproto/bsync",
+  "version": "0.0.0",
+  "license": "MIT",
+  "description": "Sychronizing service for app.bsky App View (Bluesky API)",
+  "keywords": [
+    "atproto",
+    "bluesky"
+  ],
+  "homepage": "https://atproto.com",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bluesky-social/atproto",
+    "directory": "packages/bsync"
+  },
+  "main": "dist/index.js",
+  "dependencies": {
+    "@bufbuild/protobuf": "^1.5.0",
+    "@connectrpc/connect": "^1.1.4",
+    "@connectrpc/connect-express": "^1.1.4",
+    "@connectrpc/connect-node": "^1.1.4",
+    "http-terminator": "^3.2.0",
+    "kysely": "^0.22.0",
+    "pg": "^8.10.0",
+    "pino": "^8.15.0",
+    "pino-http": "^8.2.1",
+    "typed-emitter": "^2.1.0",
+    "@atproto/common": "^0.3.3",
+    "@atproto/syntax": "^0.1.5"
+  },
+  "devDependencies": {
+    "@bufbuild/buf": "^1.28.1",
+    "@bufbuild/protoc-gen-es": "^1.5.0",
+    "@connectrpc/protoc-gen-connect-es": "^1.1.4",
+    "@types/pg": "^8.6.6"
+  },
+  "scripts": {
+    "build": "node ./build.js",
+    "postbuild": "tsc --build tsconfig.build.json",
+    "update-main-to-dist": "node ../../update-main-to-dist.js packages/bsync",
+    "start": "node --enable-source-maps dist/bin.js",
+    "test": "../dev-infra/with-test-db.sh jest",
+    "test:log": "tail -50 test.log | pino-pretty",
+    "test:updateSnapshot": "jest --updateSnapshot",
+    "migration:create": "ts-node ./bin/migration-create.ts",
+    "buf:gen": "buf generate proto"
+  },
+  "types": "dist/index.d.ts"
+}

package/proto/bsync.proto

@@ -0,0 +1,55 @@
+syntax = "proto3";
+
+package bsync;
+option go_package = "./;bsync";
+
+//
+// Sync
+//
+
+
+message MuteOperation {
+  enum Type {
+    TYPE_UNSPECIFIED = 0;
+    TYPE_ADD = 1;
+    TYPE_REMOVE = 2;
+    TYPE_CLEAR = 3;
+  }
+  string id = 1;
+  Type type = 2;
+  string actor_did = 3;
+  string subject = 4;
+}
+
+message AddMuteOperationRequest {
+  MuteOperation.Type type = 1;
+  string actor_did = 2;
+  string subject = 3;
+}
+
+message AddMuteOperationResponse {
+  MuteOperation operation = 1;
+}
+
+message ScanMuteOperationsRequest {
+  string cursor = 1;
+  int32 limit = 2;
+}
+
+message ScanMuteOperationsResponse {
+  repeated MuteOperation operations = 1;
+  string cursor = 2;
+}
+
+// Ping
+message PingRequest {}
+message PingResponse {}
+
+
+service Service {
+  // Sync
+  rpc AddMuteOperation(AddMuteOperationRequest) returns (AddMuteOperationResponse);
+  rpc ScanMuteOperations(ScanMuteOperationsRequest) returns (ScanMuteOperationsResponse);
+  // Ping
+  rpc Ping(PingRequest) returns (PingResponse);
+}

package/src/client.ts

@@ -0,0 +1,25 @@
+import {
+  Interceptor,
+  PromiseClient,
+  createPromiseClient,
+} from '@connectrpc/connect'
+import {
+  ConnectTransportOptions,
+  createConnectTransport,
+} from '@connectrpc/connect-node'
+import { Service } from './gen/bsync_connect'
+
+export type BsyncClient = PromiseClient<typeof Service>
+
+export const createClient = (opts: ConnectTransportOptions): BsyncClient => {
+  const transport = createConnectTransport(opts)
+  return createPromiseClient(Service, transport)
+}
+
+export const authWithApiKey =
+  (apiKey: string): Interceptor =>
+  (next) =>
+  (req) => {
+    req.header.set('authorization', `Bearer ${apiKey}`)
+    return next(req)
+  }

package/src/config.ts

@@ -0,0 +1,90 @@
+import assert from 'node:assert'
+import { envInt, envStr, envList, envBool } from '@atproto/common'
+
+export const envToCfg = (env: ServerEnvironment): ServerConfig => {
+  const serviceCfg: ServerConfig['service'] = {
+    port: env.port ?? 2585,
+    version: env.version ?? 'unknown',
+    longPollTimeoutMs: env.longPollTimeoutMs ?? 10000,
+  }
+
+  assert(env.dbUrl, 'missing postgres url')
+  const dbCfg: ServerConfig['db'] = {
+    url: env.dbUrl,
+    schema: env.dbSchema,
+    poolSize: env.dbPoolSize,
+    poolMaxUses: env.dbPoolMaxUses,
+    poolIdleTimeoutMs: env.dbPoolIdleTimeoutMs,
+    migrate: env.dbMigrate,
+  }
+
+  assert(env.apiKeys.length > 0, 'missing api keys')
+  const authCfg: ServerConfig['auth'] = {
+    apiKeys: new Set(env.apiKeys),
+  }
+
+  return {
+    service: serviceCfg,
+    db: dbCfg,
+    auth: authCfg,
+  }
+}
+
+export type ServerConfig = {
+  service: ServiceConfig
+  db: DatabaseConfig
+  auth: AuthConfig
+}
+
+type ServiceConfig = {
+  port: number
+  version?: string
+  longPollTimeoutMs: number
+}
+
+type DatabaseConfig = {
+  url: string
+  schema?: string
+  poolSize?: number
+  poolMaxUses?: number
+  poolIdleTimeoutMs?: number
+  migrate?: boolean
+}
+
+type AuthConfig = {
+  apiKeys: Set<string>
+}
+
+export const readEnv = (): ServerEnvironment => {
+  return {
+    // service
+    port: envInt('BSYNC_PORT'),
+    version: envStr('BSYNC_VERSION'),
+    longPollTimeoutMs: envInt('BSYNC_LONG_POLL_TIMEOUT_MS'),
+    // database
+    dbUrl: envStr('BSYNC_DB_POSTGRES_URL'),
+    dbSchema: envStr('BSYNC_DB_POSTGRES_SCHEMA'),
+    dbPoolSize: envInt('BSYNC_DB_POOL_SIZE'),
+    dbPoolMaxUses: envInt('BSYNC_DB_POOL_MAX_USES'),
+    dbPoolIdleTimeoutMs: envInt('BSYNC_DB_POOL_IDLE_TIMEOUT_MS'),
+    dbMigrate: envBool('BSYNC_DB_MIGRATE'),
+    // secrets
+    apiKeys: envList('BSYNC_API_KEYS'),
+  }
+}
+
+export type ServerEnvironment = {
+  // service
+  port?: number
+  version?: string
+  longPollTimeoutMs?: number
+  // database
+  dbUrl?: string
+  dbSchema?: string
+  dbPoolSize?: number
+  dbPoolMaxUses?: number
+  dbPoolIdleTimeoutMs?: number
+  dbMigrate?: boolean
+  // secrets
+  apiKeys: string[]
+}

package/src/context.ts

@@ -0,0 +1,42 @@
+import TypedEventEmitter from 'typed-emitter'
+import { ServerConfig } from './config'
+import Database from './db'
+import { createMuteOpChannel } from './db/schema/mute_op'
+import { EventEmitter } from 'stream'
+
+export type AppContextOptions = {
+  db: Database
+  cfg: ServerConfig
+}
+
+export class AppContext {
+  db: Database
+  cfg: ServerConfig
+  events: TypedEventEmitter<AppEvents>
+
+  constructor(opts: AppContextOptions) {
+    this.db = opts.db
+    this.cfg = opts.cfg
+    this.events = new EventEmitter() as TypedEventEmitter<AppEvents>
+  }
+
+  static async fromConfig(
+    cfg: ServerConfig,
+    overrides?: Partial<AppContextOptions>,
+  ): Promise<AppContext> {
+    const db = new Database({
+      url: cfg.db.url,
+      schema: cfg.db.schema,
+      poolSize: cfg.db.poolSize,
+      poolMaxUses: cfg.db.poolMaxUses,
+      poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs,
+    })
+    return new AppContext({ db, cfg, ...overrides })
+  }
+}
+
+export default AppContext
+
+export type AppEvents = {
+  [createMuteOpChannel]: () => void
+}

package/src/db/index.ts

@@ -0,0 +1,194 @@
+import assert from 'assert'
+import {
+  Kysely,
+  PostgresDialect,
+  Migrator,
+  KyselyPlugin,
+  PluginTransformQueryArgs,
+  PluginTransformResultArgs,
+  RootOperationNode,
+  QueryResult,
+  UnknownRow,
+} from 'kysely'
+import TypedEmitter from 'typed-emitter'
+import { Pool as PgPool, types as pgTypes } from 'pg'
+import DatabaseSchema, { DatabaseSchemaType } from './schema'
+import { PgOptions } from './types'
+import { dbLogger } from '../logger'
+import { EventEmitter } from 'stream'
+import * as migrations from './migrations'
+import { DbMigrationProvider } from './migrations/provider'
+
+export class Database {
+  pool: PgPool
+  db: DatabaseSchema
+  migrator: Migrator
+  txEvt = new EventEmitter() as TxnEmitter
+  destroyed = false
+
+  constructor(
+    public opts: PgOptions,
+    instances?: { db: DatabaseSchema; pool: PgPool },
+  ) {
+    // if instances are provided, use those
+    if (instances) {
+      this.db = instances.db
+      this.pool = instances.pool
+      return
+    }
+
+    // else create a pool & connect
+    const { schema, url } = opts
+    const pool =
+      opts.pool ??
+      new PgPool({
+        connectionString: url,
+        max: opts.poolSize,
+        maxUses: opts.poolMaxUses,
+        idleTimeoutMillis: opts.poolIdleTimeoutMs,
+      })
+
+    // Select count(*) and other pg bigints as js integer
+    pgTypes.setTypeParser(pgTypes.builtins.INT8, (n) => parseInt(n, 10))
+
+    // Setup schema usage, primarily for test parallelism (each test suite runs in its own pg schema)
+    if (schema && !/^[a-z_]+$/i.test(schema)) {
+      throw new Error(`Postgres schema must only contain [A-Za-z_]: ${schema}`)
+    }
+
+    pool.on('error', onPoolError)
+    pool.on('connect', (client) => {
+      client.on('error', onClientError)
+      if (schema) {
+        // Shared objects such as extensions will go in the public schema
+        client.query(`SET search_path TO "${schema}",public;`)
+      }
+    })
+
+    this.pool = pool
+    this.db = new Kysely<DatabaseSchemaType>({
+      dialect: new PostgresDialect({ pool }),
+    })
+    this.migrator = new Migrator({
+      db: this.db,
+      migrationTableSchema: opts.schema,
+      provider: new DbMigrationProvider(migrations),
+    })
+  }
+
+  get schema(): string | undefined {
+    return this.opts.schema
+  }
+
+  get isTransaction() {
+    return this.db.isTransaction
+  }
+
+  assertTransaction() {
+    assert(this.isTransaction, 'Transaction required')
+  }
+
+  assertNotTransaction() {
+    assert(!this.isTransaction, 'Cannot be in a transaction')
+  }
+
+  async transaction<T>(fn: (db: Database) => Promise<T>): Promise<T> {
+    const leakyTxPlugin = new LeakyTxPlugin()
+    const { dbTxn, txRes } = await this.db
+      .withPlugin(leakyTxPlugin)
+      .transaction()
+      .execute(async (txn) => {
+        const dbTxn = new Database(this.opts, {
+          db: txn,
+          pool: this.pool,
+        })
+        const txRes = await fn(dbTxn)
+          .catch(async (err) => {
+            leakyTxPlugin.endTx()
+            // ensure that all in-flight queries are flushed & the connection is open
+            await dbTxn.db.getExecutor().provideConnection(noopAsync)
+            throw err
+          })
+          .finally(() => leakyTxPlugin.endTx())
+        return { dbTxn, txRes }
+      })
+    dbTxn?.txEvt.emit('commit')
+    return txRes
+  }
+
+  onCommit(fn: () => void) {
+    this.assertTransaction()
+    this.txEvt.once('commit', fn)
+  }
+
+  async close(): Promise<void> {
+    if (this.destroyed) return
+    await this.db.destroy()
+    this.destroyed = true
+  }
+
+  async migrateToOrThrow(migration: string) {
+    if (this.schema) {
+      await this.db.schema.createSchema(this.schema).ifNotExists().execute()
+    }
+    const { error, results } = await this.migrator.migrateTo(migration)
+    if (error) {
+      throw error
+    }
+    if (!results) {
+      throw new Error('An unknown failure occurred while migrating')
+    }
+    return results
+  }
+
+  async migrateToLatestOrThrow() {
+    if (this.schema) {
+      await this.db.schema.createSchema(this.schema).ifNotExists().execute()
+    }
+    const { error, results } = await this.migrator.migrateToLatest()
+    if (error) {
+      throw error
+    }
+    if (!results) {
+      throw new Error('An unknown failure occurred while migrating')
+    }
+    return results
+  }
+}
+
+export default Database
+
+const onPoolError = (err: Error) => dbLogger.error({ err }, 'db pool error')
+const onClientError = (err: Error) => dbLogger.error({ err }, 'db client error')
+
+// utils
+// -------
+
+class LeakyTxPlugin implements KyselyPlugin {
+  private txOver: boolean
+
+  endTx() {
+    this.txOver = true
+  }
+
+  transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
+    if (this.txOver) {
+      throw new Error('tx already failed')
+    }
+    return args.node
+  }
+
+  async transformResult(
+    args: PluginTransformResultArgs,
+  ): Promise<QueryResult<UnknownRow>> {
+    return args.result
+  }
+}
+
+type TxnEmitter = TypedEmitter<TxnEvents>
+
+type TxnEvents = {
+  commit: () => void
+}
+
+const noopAsync = async () => {}

package/src/db/migrations/20240108T220751294Z-init.ts

@@ -0,0 +1,26 @@
+import { Kysely, sql } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('mute_op')
+    .addColumn('id', 'bigserial', (col) => col.primaryKey())
+    .addColumn('type', 'int2', (col) => col.notNull()) // integer enum: 0->add, 1->remove, 2->clear
+    .addColumn('actorDid', 'varchar', (col) => col.notNull())
+    .addColumn('subject', 'varchar', (col) => col.notNull())
+    .addColumn('createdAt', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`),
+    )
+    .execute()
+  await db.schema
+    .createTable('mute_item')
+    .addColumn('actorDid', 'varchar', (col) => col.notNull())
+    .addColumn('subject', 'varchar', (col) => col.notNull())
+    .addColumn('fromId', 'bigint', (col) => col.notNull())
+    .addPrimaryKeyConstraint('mute_item_pkey', ['actorDid', 'subject'])
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('mute_item').execute()
+  await db.schema.dropTable('mute_op').execute()
+}

package/src/db/migrations/index.ts

@@ -0,0 +1,5 @@
+// NOTE this file can be edited by hand, but it is also appended to by the migration:create command.
+// It's important that every migration is exported from here with the proper name. We'd simplify
+// this with kysely's FileMigrationProvider, but it doesn't play nicely with the build process.
+
+export * as _20240108T220751294Z from './20240108T220751294Z-init'

package/src/db/migrations/provider.ts

@@ -0,0 +1,8 @@
+import { Migration, MigrationProvider } from 'kysely'
+
+export class DbMigrationProvider implements MigrationProvider {
+  constructor(private migrations: Record<string, Migration>) {}
+  async getMigrations(): Promise<Record<string, Migration>> {
+    return this.migrations
+  }
+}

package/src/db/schema/index.ts

@@ -0,0 +1,9 @@
+import { Kysely } from 'kysely'
+import * as muteOp from './mute_op'
+import * as muteItem from './mute_item'
+
+export type DatabaseSchemaType = muteItem.PartialDB & muteOp.PartialDB
+
+export type DatabaseSchema = Kysely<DatabaseSchemaType>
+
+export default DatabaseSchema

package/src/db/schema/mute_item.ts

@@ -0,0 +1,13 @@
+import { Selectable } from 'kysely'
+
+export interface MuteItem {
+  actorDid: string
+  subject: string // did or aturi for list
+  fromId: number
+}
+
+export type MuteItemEntry = Selectable<MuteItem>
+
+export const tableName = 'mute_item'
+
+export type PartialDB = { [tableName]: MuteItem }

package/src/db/schema/mute_op.ts

@@ -0,0 +1,18 @@
+import { GeneratedAlways, Selectable } from 'kysely'
+import { MuteOperation_Type } from '../../gen/bsync_pb'
+
+export interface MuteOp {
+  id: GeneratedAlways<number>
+  type: MuteOperation_Type // integer enum: 0->add, 1->remove, 2->clear
+  actorDid: string
+  subject: string // did or aturi for list
+  createdAt: GeneratedAlways<Date>
+}
+
+export type MuteOpEntry = Selectable<MuteOp>
+
+export const tableName = 'mute_op'
+
+export type PartialDB = { [tableName]: MuteOp }
+
+export const createMuteOpChannel = 'mute_op_create' // used with listen/notify

package/src/db/types.ts

@@ -0,0 +1,15 @@
+import { Pool as PgPool } from 'pg'
+import { DynamicModule, RawBuilder, SelectQueryBuilder } from 'kysely'
+
+export type DbRef = RawBuilder | ReturnType<DynamicModule['ref']>
+
+export type AnyQb = SelectQueryBuilder<any, any, any>
+
+export type PgOptions = {
+  url: string
+  pool?: PgPool
+  schema?: string
+  poolSize?: number
+  poolMaxUses?: number
+  poolIdleTimeoutMs?: number
+}

package/src/gen/bsync_connect.ts

@@ -0,0 +1,54 @@
+// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension=.ts"
+// @generated from file bsync.proto (package bsync, syntax proto3)
+/* eslint-disable */
+// @ts-nocheck
+
+import {
+  AddMuteOperationRequest,
+  AddMuteOperationResponse,
+  PingRequest,
+  PingResponse,
+  ScanMuteOperationsRequest,
+  ScanMuteOperationsResponse,
+} from './bsync_pb.ts'
+import { MethodKind } from '@bufbuild/protobuf'
+
+/**
+ * @generated from service bsync.Service
+ */
+export const Service = {
+  typeName: 'bsync.Service',
+  methods: {
+    /**
+     * Sync
+     *
+     * @generated from rpc bsync.Service.AddMuteOperation
+     */
+    addMuteOperation: {
+      name: 'AddMuteOperation',
+      I: AddMuteOperationRequest,
+      O: AddMuteOperationResponse,
+      kind: MethodKind.Unary,
+    },
+    /**
+     * @generated from rpc bsync.Service.ScanMuteOperations
+     */
+    scanMuteOperations: {
+      name: 'ScanMuteOperations',
+      I: ScanMuteOperationsRequest,
+      O: ScanMuteOperationsResponse,
+      kind: MethodKind.Unary,
+    },
+    /**
+     * Ping
+     *
+     * @generated from rpc bsync.Service.Ping
+     */
+    ping: {
+      name: 'Ping',
+      I: PingRequest,
+      O: PingResponse,
+      kind: MethodKind.Unary,
+    },
+  },
+} as const