@atproto/pds 0.4.33 → 0.4.35

package/dist/sequencer/events.d.ts

@@ -47,9 +47,9 @@ export declare const commitEvt: z.ZodObject<{
     }>, "many">;
     blobs: z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>, "many">;
 }, "strip", z.ZodTypeAny, {
-    rev: string;
     repo: string;
     blobs: CID[];
+    rev: string;
     rebase: boolean;
     tooBig: boolean;
     commit: CID;
@@ -62,9 +62,9 @@ export declare const commitEvt: z.ZodObject<{
     }[];
     prev: CID | null;
 }, {
-    rev: string;
     repo: string;
     blobs: any[];
+    rev: string;
     rebase: boolean;
     tooBig: boolean;
     since: string | null;

package/example.env

@@ -1,10 +1,10 @@
 # See more env options in src/config/env.ts
 # Hostname - the public domain that you intend to deploy your service at
 PDS_HOSTNAME="example.com"
+PDS_PORT="2583"
 
 # Database config - use one or the other
-PDS_DB_SQLITE_LOCATION="db.test"
-# PDS_DB_POSTGRES_URL="postgresql://pg:password@localhost:5433/postgres"
+PDS_DATA_DIRECTORY="data"
 
 # Blobstore - filesystem location to store uploaded blobs
 PDS_BLOBSTORE_DISK_LOCATION="blobs"
@@ -14,6 +14,7 @@ PDS_REPO_SIGNING_KEY_K256_PRIVATE_KEY_HEX="3ee68..."
 PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX="e049f..."
 
 # Secrets - update to secure high-entropy strings
+PDS_DPOP_SECRET="32-random-bytes-hex-encoded"
 PDS_JWT_SECRET="jwt-secret"
 PDS_ADMIN_PASSWORD="admin-pass"
 
@@ -21,4 +22,21 @@ PDS_ADMIN_PASSWORD="admin-pass"
 PDS_DID_PLC_URL="https://plc.bsky-sandbox.dev"
 PDS_BSKY_APP_VIEW_ENDPOINT="https://api.bsky-sandbox.dev"
 PDS_BSKY_APP_VIEW_DID="did:web:api.bsky-sandbox.dev"
-PDS_CRAWLERS="https://bgs.bsky-sandbox.dev"
+PDS_CRAWLERS="https://bgs.bsky-sandbox.dev"
+
+# OAuth Provider
+PDS_OAUTH_PROVIDER_NAME="John's self hosted PDS"
+PDS_OAUTH_PROVIDER_LOGO=
+PDS_OAUTH_PROVIDER_PRIMARY_COLOR="#7507e3"
+PDS_OAUTH_PROVIDER_ERROR_COLOR=
+PDS_OAUTH_PROVIDER_HOME_LINK=
+PDS_OAUTH_PROVIDER_TOS_LINK=
+PDS_OAUTH_PROVIDER_POLICY_LINK=
+PDS_OAUTH_PROVIDER_SUPPORT_LINK=
+
+# Debugging
+NODE_TLS_REJECT_UNAUTHORIZED=1
+LOG_ENABLED=0
+LOG_LEVEL=info
+PDS_INVITE_REQUIRED=1
+PDS_DISABLE_SSRF_PROTECTION=0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/pds",
-  "version": "0.4.33",
+  "version": "0.4.35",
   "license": "MIT",
   "description": "Reference implementation of atproto Personal Data Server (PDS)",
   "keywords": [
@@ -18,7 +18,7 @@
   "bin": "dist/bin.js",
   "dependencies": {
     "@did-plc/lib": "^0.0.4",
-    "better-sqlite3": "^9.4.0",
+    "better-sqlite3": "^10.0.0",
     "bytes": "^3.1.2",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
@@ -37,18 +37,20 @@
     "nodemailer": "^6.8.0",
     "nodemailer-html-to-text": "^3.2.0",
     "p-queue": "^6.6.2",
-    "pino": "^8.15.0",
+    "pino": "^8.21.0",
     "pino-http": "^8.2.1",
     "sharp": "^0.32.6",
     "typed-emitter": "^2.1.0",
     "uint8arrays": "3.0.0",
-    "zod": "^3.21.4",
-    "@atproto/api": "^0.12.18",
+    "zod": "^3.23.8",
+    "@atproto-labs/fetch-node": "0.1.0",
+    "@atproto/api": "^0.12.19",
     "@atproto/aws": "^0.2.0",
     "@atproto/common": "^0.4.0",
     "@atproto/crypto": "^0.4.0",
     "@atproto/identity": "^0.4.0",
     "@atproto/lexicon": "^0.4.0",
+    "@atproto/oauth-provider": "^0.1.0",
     "@atproto/repo": "^0.4.0",
     "@atproto/syntax": "^0.3.0",
     "@atproto/xrpc": "^0.5.0",
@@ -70,8 +72,8 @@
     "jest": "^28.1.2",
     "ts-node": "^10.8.2",
     "ws": "^8.12.0",
-    "@atproto/api": "^0.12.18",
-    "@atproto/bsky": "^0.0.61",
+    "@atproto/api": "^0.12.19",
+    "@atproto/bsky": "^0.0.62",
     "@atproto/lex-cli": "^0.4.0"
   },
   "scripts": {

package/src/account-manager/db/migrations/004-oauth.ts

@@ -0,0 +1,122 @@
+import { Kysely, sql } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createTable('authorization_request')
+    .addColumn('id', 'varchar', (col) => col.primaryKey())
+    .addColumn('did', 'varchar')
+    .addColumn('deviceId', 'varchar')
+    .addColumn('clientId', 'varchar', (col) => col.notNull())
+    .addColumn('clientAuth', 'varchar', (col) => col.notNull())
+    .addColumn('parameters', 'varchar', (col) => col.notNull())
+    .addColumn('expiresAt', 'varchar', (col) => col.notNull())
+    .addColumn('code', 'varchar')
+    .execute()
+
+  await db.schema
+    .createIndex('authorization_request_code_idx')
+    .unique()
+    .on('authorization_request')
+    // https://github.com/kysely-org/kysely/issues/302
+    .expression(sql`code DESC) WHERE (code IS NOT NULL`)
+    .execute()
+
+  await db.schema
+    .createIndex('authorization_request_expires_at_idx')
+    .on('authorization_request')
+    .column('expiresAt')
+    .execute()
+
+  await db.schema
+    .createTable('device')
+    .addColumn('id', 'varchar', (col) => col.primaryKey())
+    .addColumn('sessionId', 'varchar', (col) => col.notNull())
+    .addColumn('userAgent', 'varchar')
+    .addColumn('ipAddress', 'varchar', (col) => col.notNull())
+    .addColumn('lastSeenAt', 'varchar', (col) => col.notNull())
+    .addUniqueConstraint('device_session_id_idx', ['sessionId'])
+    .execute()
+
+  await db.schema
+    .createTable('device_account')
+    .addColumn('did', 'varchar', (col) => col.notNull())
+    .addColumn('deviceId', 'varchar', (col) => col.notNull())
+    .addColumn('authenticatedAt', 'varchar', (col) => col.notNull())
+    .addColumn('remember', 'boolean', (col) => col.notNull())
+    .addColumn('authorizedClients', 'varchar', (col) => col.notNull())
+    .addPrimaryKeyConstraint('device_account_pk', [
+      'deviceId', // first because this table will be joined from the "device" table
+      'did',
+    ])
+    .addForeignKeyConstraint(
+      'device_account_device_id_fk',
+      ['deviceId'],
+      'device',
+      ['id'],
+      (qb) => qb.onDelete('cascade').onUpdate('cascade'),
+    )
+    .execute()
+
+  await db.schema
+    .createTable('token')
+    .addColumn('id', 'integer', (col) => col.primaryKey().autoIncrement())
+    .addColumn('did', 'varchar', (col) => col.notNull())
+    .addColumn('tokenId', 'varchar', (col) => col.notNull())
+    .addColumn('createdAt', 'varchar', (col) => col.notNull())
+    .addColumn('updatedAt', 'varchar', (col) => col.notNull())
+    .addColumn('expiresAt', 'varchar', (col) => col.notNull())
+    .addColumn('clientId', 'varchar', (col) => col.notNull())
+    .addColumn('clientAuth', 'varchar', (col) => col.notNull())
+    .addColumn('deviceId', 'varchar')
+    .addColumn('parameters', 'varchar', (col) => col.notNull())
+    .addColumn('details', 'varchar')
+    .addColumn('code', 'varchar')
+    .addColumn('currentRefreshToken', 'varchar')
+    .addUniqueConstraint('token_current_refresh_token_unique_idx', [
+      'currentRefreshToken',
+    ])
+    .addUniqueConstraint('token_id_unique_idx', ['tokenId'])
+    .execute()
+
+  await db.schema
+    .createIndex('token_did_idx')
+    .on('token')
+    .column('did')
+    .execute()
+
+  await db.schema
+    .createIndex('token_code_idx')
+    .unique()
+    .on('token')
+    // https://github.com/kysely-org/kysely/issues/302
+    .expression(sql`code DESC) WHERE (code IS NOT NULL`)
+    .execute()
+
+  await db.schema
+    .createTable('used_refresh_token')
+    .addColumn('refreshToken', 'varchar', (col) => col.primaryKey())
+    .addColumn('tokenId', 'integer', (col) => col.notNull())
+    .addForeignKeyConstraint(
+      'used_refresh_token_fk',
+      ['tokenId'],
+      'token',
+      ['id'],
+      // uses "used_refresh_token_id_idx" index (when cascading)
+      (qb) => qb.onDelete('cascade').onUpdate('cascade'),
+    )
+    .execute()
+
+  await db.schema
+    .createIndex('used_refresh_token_id_idx')
+    .on('used_refresh_token')
+    .column('tokenId')
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropTable('used_refresh_token').execute()
+  await db.schema.dropTable('token').execute()
+  await db.schema.dropTable('device_account').execute()
+  await db.schema.dropTable('device').execute()
+  await db.schema.dropTable('authorization_request').execute()
+}

package/src/account-manager/db/migrations/index.ts

@@ -1,9 +1,11 @@
 import * as mig001 from './001-init'
 import * as mig002 from './002-account-deactivation'
 import * as mig003 from './003-privileged-app-passwords'
+import * as mig004 from './004-oauth'
 
 export default {
   '001': mig001,
   '002': mig002,
   '003': mig003,
+  '004': mig004,
 }

package/src/account-manager/db/schema/authorization-request.ts

@@ -0,0 +1,26 @@
+import {
+  Code,
+  DeviceId,
+  OAuthClientId,
+  RequestId,
+} from '@atproto/oauth-provider'
+import { Selectable } from 'kysely'
+import { DateISO, JsonObject } from '../../../db'
+
+export interface AuthorizationRequest {
+  id: RequestId
+  did: string | null
+  deviceId: DeviceId | null
+
+  clientId: OAuthClientId
+  clientAuth: JsonObject
+  parameters: JsonObject
+  expiresAt: DateISO
+  code: Code | null
+}
+
+export type AuthorizationRequestEntry = Selectable<AuthorizationRequest>
+
+export const tableName = 'authorization_request'
+
+export type PartialDB = { [tableName]: AuthorizationRequest }

package/src/account-manager/db/schema/device-account.ts

@@ -0,0 +1,15 @@
+import { DeviceId } from '@atproto/oauth-provider'
+import { DateISO, JsonArray } from '../../../db'
+
+export interface DeviceAccount {
+  did: string
+  deviceId: DeviceId
+
+  authenticatedAt: DateISO
+  authorizedClients: JsonArray
+  remember: 0 | 1
+}
+
+export const tableName = 'device_account'
+
+export type PartialDB = { [tableName]: DeviceAccount }

package/src/account-manager/db/schema/device.ts

@@ -0,0 +1,18 @@
+import { DeviceId, SessionId } from '@atproto/oauth-provider'
+import { Selectable } from 'kysely'
+import { DateISO } from '../../../db'
+
+export interface Device {
+  id: DeviceId
+  sessionId: SessionId
+
+  userAgent: string | null
+  ipAddress: string
+  lastSeenAt: DateISO
+}
+
+export type DeviceEntry = Selectable<Device>
+
+export const tableName = 'device'
+
+export type PartialDB = { [tableName]: Device }

package/src/account-manager/db/schema/index.ts

@@ -1,5 +1,10 @@
 import * as actor from './actor'
 import * as account from './account'
+import * as device from './device'
+import * as deviceAccount from './device-account'
+import * as oauthRequest from './authorization-request'
+import * as token from './token'
+import * as usedRefreshToken from './used-refresh-token'
 import * as repoRoot from './repo-root'
 import * as refreshToken from './refresh-token'
 import * as appPassword from './app-password'
@@ -8,6 +13,11 @@ import * as emailToken from './email-token'
 
 export type DatabaseSchema = actor.PartialDB &
   account.PartialDB &
+  device.PartialDB &
+  deviceAccount.PartialDB &
+  oauthRequest.PartialDB &
+  token.PartialDB &
+  usedRefreshToken.PartialDB &
   refreshToken.PartialDB &
   appPassword.PartialDB &
   repoRoot.PartialDB &
@@ -16,6 +26,11 @@ export type DatabaseSchema = actor.PartialDB &
 
 export type { Actor, ActorEntry } from './actor'
 export type { Account, AccountEntry } from './account'
+export type { Device } from './device'
+export type { DeviceAccount } from './device-account'
+export type { AuthorizationRequest } from './authorization-request'
+export type { Token } from './token'
+export type { UsedRefreshToken } from './used-refresh-token'
 export type { RepoRoot } from './repo-root'
 export type { RefreshToken } from './refresh-token'
 export type { AppPassword } from './app-password'

package/src/account-manager/db/schema/token.ts

@@ -0,0 +1,34 @@
+import {
+  Code,
+  DeviceId,
+  OAuthClientId,
+  RefreshToken,
+  Sub,
+  TokenId,
+} from '@atproto/oauth-provider'
+import { Generated, Selectable } from 'kysely'
+
+import { DateISO, JsonArray, JsonObject } from '../../../db/cast'
+
+export interface Token {
+  id: Generated<number>
+  did: Sub
+
+  tokenId: TokenId
+  createdAt: DateISO
+  updatedAt: DateISO
+  expiresAt: DateISO
+  clientId: OAuthClientId
+  clientAuth: JsonObject
+  deviceId: DeviceId | null
+  parameters: JsonObject
+  details: JsonArray | null
+  code: Code | null
+  currentRefreshToken: RefreshToken | null
+}
+
+export type TokenEntry = Selectable<Token>
+
+export const tableName = 'token'
+
+export type PartialDB = { [tableName]: Token }

package/src/account-manager/db/schema/used-refresh-token.ts

@@ -0,0 +1,13 @@
+import { RefreshToken } from '@atproto/oauth-provider'
+import { Selectable } from 'kysely'
+
+export interface UsedRefreshToken {
+  tokenId: number
+  refreshToken: RefreshToken
+}
+
+export type UsedRefreshTokenEntry = Selectable<UsedRefreshToken>
+
+export const tableName = 'used_refresh_token'
+
+export type PartialDB = { [tableName]: UsedRefreshToken }

package/src/account-manager/helpers/account.ts

@@ -9,8 +9,6 @@ export type ActorAccount = ActorEntry & {
   email: string | null
   emailConfirmedAt: string | null
   invitesDisabled: 0 | 1 | null
-  active: boolean
-  status?: AccountStatus
 }
 
 export type AvailabilityFlags = {
@@ -26,7 +24,7 @@ export enum AccountStatus {
   Deactivated = 'deactivated',
 }
 
-const selectAccountQB = (db: AccountDb, flags?: AvailabilityFlags) => {
+export const selectAccountQB = (db: AccountDb, flags?: AvailabilityFlags) => {
   const { includeTakenDown = false, includeDeactivated = false } = flags ?? {}
   const { ref } = db.db.dynamic
   return db.db
@@ -63,7 +61,7 @@ export const getAccount = async (
       }
     })
     .executeTakeFirst()
-  return found ? { ...found, ...formatAccountStatus(found) } : null
+  return found || null
 }
 
 export const getAccountByEmail = async (
@@ -74,7 +72,7 @@ export const getAccountByEmail = async (
   const found = await selectAccountQB(db, flags)
     .where('email', '=', email.toLowerCase())
     .executeTakeFirst()
-  return found ? { ...found, ...formatAccountStatus(found) } : null
+  return found || null
 }
 
 export const registerActor = async (
@@ -267,22 +265,19 @@ export const activateAccount = async (db: AccountDb, did: string) => {
   )
 }
 
-export const formatAccountStatus = (account: {
-  takedownRef: string | null
-  deactivatedAt: string | null
-}): {
-  active: boolean
-  status?: AccountStatus
-} => {
-  let status: AccountStatus | undefined = undefined
-  if (account.takedownRef) {
-    status = AccountStatus.Takendown
+export const formatAccountStatus = (
+  account: null | {
+    takedownRef: string | null
+    deactivatedAt: string | null
+  },
+) => {
+  if (!account) {
+    return { active: false, status: AccountStatus.Deleted } as const
+  } else if (account.takedownRef) {
+    return { active: false, status: AccountStatus.Takendown } as const
   } else if (account.deactivatedAt) {
-    status = AccountStatus.Deactivated
-  }
-  const active = !status
-  return {
-    active,
-    status,
+    return { active: false, status: AccountStatus.Deactivated } as const
+  } else {
+    return { active: true, status: undefined } as const
   }
 }

package/src/account-manager/helpers/authorization-request.ts

@@ -0,0 +1,82 @@
+import {
+  Code,
+  FoundRequestResult,
+  RequestData,
+  RequestId,
+  UpdateRequestData,
+} from '@atproto/oauth-provider'
+import { AccountDb, AuthorizationRequest } from '../db'
+import { fromDateISO, fromJsonObject, toDateISO, toJsonObject } from '../../db'
+import { Insertable, Selectable } from 'kysely'
+
+export const rowToRequestData = (
+  row: Selectable<AuthorizationRequest>,
+): RequestData => ({
+  clientId: row.clientId,
+  clientAuth: fromJsonObject(row.clientAuth),
+  parameters: fromJsonObject(row.parameters),
+  expiresAt: fromDateISO(row.expiresAt),
+  deviceId: row.deviceId,
+  sub: row.did,
+  code: row.code,
+})
+
+export const rowToFoundRequestResult = (
+  row: Selectable<AuthorizationRequest>,
+): FoundRequestResult => ({
+  id: row.id,
+  data: rowToRequestData(row),
+})
+
+const requestDataToRow = (
+  id: RequestId,
+  data: RequestData,
+): Insertable<AuthorizationRequest> => ({
+  id,
+  did: data.sub,
+  deviceId: data.deviceId,
+
+  clientId: data.clientId,
+  clientAuth: toJsonObject(data.clientAuth),
+  parameters: toJsonObject(data.parameters),
+  expiresAt: toDateISO(data.expiresAt),
+  code: data.code,
+})
+
+export const createQB = (db: AccountDb, id: RequestId, data: RequestData) =>
+  db.db.insertInto('authorization_request').values(requestDataToRow(id, data))
+
+export const readQB = (db: AccountDb, id: RequestId) =>
+  db.db.selectFrom('authorization_request').where('id', '=', id).selectAll()
+
+export const updateQB = (
+  db: AccountDb,
+  id: RequestId,
+  { code, sub, deviceId, expiresAt }: UpdateRequestData,
+) =>
+  db.db
+    .updateTable('authorization_request')
+    .if(code !== undefined, (qb) => qb.set({ code }))
+    .if(sub !== undefined, (qb) => qb.set({ did: sub }))
+    .if(deviceId !== undefined, (qb) => qb.set({ deviceId }))
+    .if(expiresAt != null, (qb) => qb.set({ expiresAt: toDateISO(expiresAt!) }))
+    .where('id', '=', id)
+
+export const removeOldExpiredQB = (db: AccountDb, delay = 600e3) =>
+  // We allow some delay for the expiration time so that expired requests
+  // can still be returned to the OAuthProvider library for error handling.
+  db.db
+    .deleteFrom('authorization_request')
+    // uses "authorization_request_expires_at_idx" index
+    .where('expiresAt', '<', toDateISO(new Date(Date.now() - delay)))
+
+export const removeByIdQB = (db: AccountDb, id: RequestId) =>
+  db.db.deleteFrom('authorization_request').where('id', '=', id)
+
+export const findByCodeQB = (db: AccountDb, code: Code) =>
+  db.db
+    .selectFrom('authorization_request')
+    // uses "authorization_request_code_idx" partial index (hence the null check)
+    .where('code', '=', code)
+    .where('code', 'is not', null)
+    .selectAll()