@atproto/tap 0.0.2

package/src/channel.ts

@@ -0,0 +1,152 @@
+import { ClientOptions } from 'ws'
+import { Deferrable, createDeferrable, isErrnoException } from '@atproto/common'
+import { WebSocketKeepAlive } from '@atproto/ws-client'
+import { TapEvent, parseTapEvent } from './types'
+import { formatAdminAuthHeader } from './util'
+
+export interface HandlerOpts {
+  signal: AbortSignal
+  ack: () => Promise<void>
+}
+
+export interface TapHandler {
+  onEvent: (evt: TapEvent, opts: HandlerOpts) => void | Promise<void>
+  onError: (err: Error) => void
+}
+
+export type TapWebsocketOptions = ClientOptions & {
+  adminPassword?: string
+  maxReconnectSeconds?: number
+  heartbeatIntervalMs?: number
+  onReconnectError?: (error: unknown, n: number, initialSetup: boolean) => void
+}
+
+type BufferedAck = {
+  id: number
+  defer: Deferrable
+}
+
+export class TapChannel {
+  private ws: WebSocketKeepAlive
+  private handler: TapHandler
+
+  private readonly abortController: AbortController = new AbortController()
+  private readonly destroyDefer: Deferrable = createDeferrable()
+
+  private bufferedAcks: BufferedAck[] = []
+
+  constructor(
+    url: string,
+    handler: TapHandler,
+    wsOpts: TapWebsocketOptions = {},
+  ) {
+    this.handler = handler
+    const { adminPassword, ...rest } = wsOpts
+    let headers = rest.headers
+    if (adminPassword) {
+      headers ??= {}
+      headers['Authorization'] = formatAdminAuthHeader(adminPassword)
+    }
+    this.ws = new WebSocketKeepAlive({
+      getUrl: async () => url,
+      onReconnect: () => {
+        this.flushBufferedAcks()
+      },
+      signal: this.abortController.signal,
+      ...rest,
+      headers,
+    })
+  }
+
+  async ackEvent(id: number): Promise<void> {
+    if (this.ws.isConnected()) {
+      try {
+        await this.sendAck(id)
+      } catch {
+        await this.bufferAndSendAck(id)
+      }
+    } else {
+      await this.bufferAndSendAck(id)
+    }
+  }
+
+  private async sendAck(id: number): Promise<void> {
+    await this.ws.send(JSON.stringify({ type: 'ack', id }))
+  }
+
+  // resolves after the ack has been actually sent
+  private async bufferAndSendAck(id: number): Promise<void> {
+    const defer = createDeferrable()
+    this.bufferedAcks.push({
+      id,
+      defer,
+    })
+    await defer.complete
+  }
+
+  private async flushBufferedAcks(): Promise<void> {
+    while (this.bufferedAcks.length > 0) {
+      try {
+        const ack = this.bufferedAcks.at(0)
+        if (!ack) {
+          return
+        }
+        await this.sendAck(ack.id)
+        ack.defer.resolve()
+        this.bufferedAcks = this.bufferedAcks.slice(1)
+      } catch (err) {
+        this.handler.onError(
+          new Error(`failed to send ack for event ${this.bufferedAcks[0]}`, {
+            cause: err,
+          }),
+        )
+        return
+      }
+    }
+  }
+
+  async start() {
+    try {
+      for await (const chunk of this.ws) {
+        await this.processWsEvent(chunk)
+      }
+    } catch (err) {
+      if (isErrnoException(err) && err.name === 'AbortError') {
+        this.destroyDefer.resolve()
+      } else {
+        throw err
+      }
+    }
+  }
+
+  private async processWsEvent(chunk: Uint8Array) {
+    let evt: TapEvent
+    try {
+      const data = chunk.toString()
+      evt = parseTapEvent(JSON.parse(data))
+    } catch (err) {
+      this.handler.onError(new Error('Failed to parse message', { cause: err }))
+      return
+    }
+
+    try {
+      await this.handler.onEvent(evt, {
+        signal: this.abortController.signal,
+        ack: async () => {
+          await this.ackEvent(evt.id)
+        },
+      })
+    } catch (err) {
+      // Don't ack on error - let Tap retry
+      this.handler.onError(
+        new Error(`Failed to process event ${evt.id}`, { cause: err }),
+      )
+      return
+    }
+  }
+
+  async destroy(): Promise<void> {
+    this.abortController.abort()
+    await this.destroyDefer.complete
+  }
+}

package/src/client.ts

@@ -0,0 +1,100 @@
+import { DidDocument, didDocument } from '@atproto/common'
+import { TapChannel, TapHandler, TapWebsocketOptions } from './channel'
+import { RepoInfo, repoInfoSchema } from './types'
+import { formatAdminAuthHeader } from './util'
+
+export interface TapConfig {
+  adminPassword?: string
+}
+
+export class Tap {
+  url: string
+  private adminPassword?: string
+  private authHeader?: string
+
+  constructor(url: string, config: TapConfig = {}) {
+    if (!url.startsWith('http://') && !url.startsWith('https://')) {
+      throw new Error('Invalid URL, expected http:// or https://')
+    }
+    this.url = url
+    this.adminPassword = config.adminPassword
+    if (this.adminPassword) {
+      this.authHeader = formatAdminAuthHeader(this.adminPassword)
+    }
+  }
+
+  private getHeaders(): Record<string, string> {
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+    }
+    if (this.authHeader) {
+      headers['Authorization'] = this.authHeader
+    }
+    return headers
+  }
+
+  channel(handler: TapHandler, opts?: TapWebsocketOptions): TapChannel {
+    const url = new URL(this.url)
+    url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'
+    url.pathname = '/channel'
+    return new TapChannel(url.toString(), handler, {
+      adminPassword: this.adminPassword,
+      ...opts,
+    })
+  }
+
+  async addRepos(dids: string[]): Promise<void> {
+    const response = await fetch(`${this.url}/repos/add`, {
+      method: 'POST',
+      headers: this.getHeaders(),
+      body: JSON.stringify({ dids }),
+    })
+    await response.body?.cancel() // expect empty body
+
+    if (!response.ok) {
+      throw new Error(`Failed to add repos: ${response.statusText}`)
+    }
+  }
+
+  async removeRepos(dids: string[]): Promise<void> {
+    const response = await fetch(`${this.url}/repos/remove`, {
+      method: 'POST',
+      headers: this.getHeaders(),
+      body: JSON.stringify({ dids }),
+    })
+    await response.body?.cancel() // expect empty body
+
+    if (!response.ok) {
+      throw new Error(`Failed to remove repos: ${response.statusText}`)
+    }
+  }
+
+  async resolveDid(did: string): Promise<DidDocument | null> {
+    const response = await fetch(`${this.url}/resolve/${did}`, {
+      method: 'GET',
+      headers: this.getHeaders(),
+    })
+
+    if (response.status === 404) {
+      return null
+    } else if (!response.ok) {
+      await response.body?.cancel()
+      throw new Error(`Failed to resolve DID: ${response.statusText}`)
+    }
+    return didDocument.parse(await response.json())
+  }
+
+  async getRepoInfo(did: string): Promise<RepoInfo> {
+    const response = await fetch(`${this.url}/info/${did}`, {
+      method: 'GET',
+      headers: this.getHeaders(),
+    })
+
+    if (!response.ok) {
+      await response.body?.cancel()
+      throw new Error(`Failed to get repo info: ${response.statusText}`)
+    }
+
+    return repoInfoSchema.parse(await response.json())
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './types'
+export * from './client'
+export * from './channel'
+export * from './simple-indexer'
+export * from './util'

package/src/simple-indexer.ts

@@ -0,0 +1,47 @@
+import { HandlerOpts, TapHandler } from './channel'
+import { IdentityEvent, RecordEvent, TapEvent } from './types'
+
+type IdentityEventHandler = (
+  evt: IdentityEvent,
+  opts?: HandlerOpts,
+) => Promise<void>
+type RecordEventHandler = (
+  evt: RecordEvent,
+  opts?: HandlerOpts,
+) => Promise<void>
+type ErrorHandler = (err: Error) => void
+
+export class SimpleIndexer implements TapHandler {
+  private identityHandler: IdentityEventHandler | undefined
+  private recordHandler: RecordEventHandler | undefined
+  private errorHandler: ErrorHandler | undefined
+
+  identity(fn: IdentityEventHandler) {
+    this.identityHandler = fn
+  }
+
+  record(fn: RecordEventHandler) {
+    this.recordHandler = fn
+  }
+
+  error(fn: ErrorHandler) {
+    this.errorHandler = fn
+  }
+
+  async onEvent(evt: TapEvent, opts: HandlerOpts): Promise<void> {
+    if (evt.type === 'record') {
+      await this.recordHandler?.(evt, opts)
+    } else {
+      await this.identityHandler?.(evt, opts)
+    }
+    await opts.ack()
+  }
+
+  onError(err: Error) {
+    if (this.errorHandler) {
+      this.errorHandler(err)
+    } else {
+      throw err
+    }
+  }
+}

package/src/types.ts

@@ -0,0 +1,109 @@
+import { z } from 'zod'
+
+export const recordEventDataSchema = z.object({
+  did: z.string(),
+  rev: z.string(),
+  collection: z.string(),
+  rkey: z.string(),
+  action: z.enum(['create', 'update', 'delete']),
+  record: z.record(z.string(), z.unknown()).optional(),
+  cid: z.string().optional(),
+  live: z.boolean(),
+})
+
+export const identityEventDataSchema = z.object({
+  did: z.string(),
+  handle: z.string(),
+  is_active: z.boolean(),
+  status: z.enum([
+    'active',
+    'takendown',
+    'suspended',
+    'deactivated',
+    'deleted',
+  ]),
+})
+
+export const recordEventSchema = z.object({
+  id: z.number(),
+  type: z.literal('record'),
+  record: recordEventDataSchema,
+})
+
+export const identityEventSchema = z.object({
+  id: z.number(),
+  type: z.literal('identity'),
+  identity: identityEventDataSchema,
+})
+
+export const tapEventSchema = z.union([recordEventSchema, identityEventSchema])
+
+export type RecordEvent = {
+  id: number
+  type: 'record'
+  action: 'create' | 'update' | 'delete'
+  did: string
+  rev: string
+  collection: string
+  rkey: string
+  record?: Record<string, unknown>
+  cid?: string
+  live: boolean
+}
+
+export type IdentityEvent = {
+  id: number
+  type: 'identity'
+  did: string
+  handle: string
+  isActive: boolean
+  status: RepoStatus
+}
+
+export type RepoStatus =
+  | 'active'
+  | 'takendown'
+  | 'suspended'
+  | 'deactivated'
+  | 'deleted'
+
+export type TapEvent = IdentityEvent | RecordEvent
+
+export const parseTapEvent = (data: unknown): TapEvent => {
+  const parsed = tapEventSchema.parse(data)
+  if (parsed.type === 'identity') {
+    return {
+      id: parsed.id,
+      type: parsed.type,
+      did: parsed.identity.did,
+      handle: parsed.identity.handle,
+      isActive: parsed.identity.is_active,
+      status: parsed.identity.status,
+    }
+  } else {
+    return {
+      id: parsed.id,
+      type: parsed.type,
+      action: parsed.record.action,
+      did: parsed.record.did,
+      rev: parsed.record.rev,
+      collection: parsed.record.collection,
+      rkey: parsed.record.rkey,
+      record: parsed.record.record,
+      cid: parsed.record.cid,
+      live: parsed.record.live,
+    }
+  }
+}
+
+export const repoInfoSchema = z.object({
+  did: z.string(),
+  handle: z.string(),
+  state: z.string(),
+  rev: z.string(),
+  records: z.number(),
+  error: z.string().optional(),
+  retries: z.number().optional(),
+})
+
+export type RepoInfo = z.infer<typeof repoInfoSchema>

package/src/util.ts

@@ -0,0 +1,33 @@
+export const formatAdminAuthHeader = (password: string) => {
+  return 'Basic ' + Buffer.from(`admin:${password}`).toString('base64')
+}
+
+export const parseAdminAuthHeader = (header: string) => {
+  const noPrefix = header.startsWith('Basic ') ? header.slice(6) : header
+  const [username, password] = Buffer.from(noPrefix, 'base64')
+    .toString()
+    .split(':')
+  if (username !== 'admin') {
+    throw new Error("Unexpected username in admin headers. Expected 'admin'")
+  }
+  return password
+}
+
+export const assureAdminAuth = (expectedPassword: string, header: string) => {
+  const headerPassword = parseAdminAuthHeader(header)
+  const passEqual = timingSafeEqual(headerPassword, expectedPassword)
+  if (!passEqual) {
+    throw new Error('Invalid admin password')
+  }
+}
+
+const timingSafeEqual = (a: string, b: string): boolean => {
+  const bufA = Buffer.from(a)
+  const bufB = Buffer.from(b)
+  if (bufA.length !== bufB.length) {
+    // Compare against self to maintain constant time even with length mismatch
+    Buffer.from(a).compare(Buffer.from(a))
+    return false
+  }
+  return bufA.compare(bufB) === 0
+}