@atproto/bsync 0.0.32 → 0.0.33

package/src/routes/delete-operations.ts

@@ -1,45 +0,0 @@
-import { Code, ConnectError, ServiceImpl } from '@connectrpc/connect'
-import { AppContext } from '../context.js'
-import { Service } from '../proto/bsync_connect.js'
-import { DeleteOperationsByActorAndNamespaceResponse } from '../proto/bsync_pb.js'
-import { authWithApiKey } from './auth.js'
-import { isValidDid, validateNamespace } from './util.js'
-
-export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
-  /**
-   * This method is responsible for deleting log rows from the bsync db, it has
-   * no other downstream effects. This method is called from the dataplane in
-   * response to a data deletion request initiated by a moderator in Ozone.
-   * It's the final step of the deletion process, basically cleaning up the
-   * breadcrumbs that resulted in the state we store in the dataplane.
-   */
-  async deleteOperationsByActorAndNamespace(req, handlerCtx) {
-    authWithApiKey(ctx, handlerCtx)
-    const { db } = ctx
-
-    try {
-      validateNamespace(req.namespace)
-    } catch (error) {
-      throw new ConnectError(
-        'requested namespace for deletion is invalid NSID',
-        Code.InvalidArgument,
-      )
-    }
-    if (!isValidDid(req.actorDid)) {
-      throw new ConnectError(
-        'requested actor_did for deletion is invalid DID',
-        Code.InvalidArgument,
-      )
-    }
-
-    const deletedRows = await db.db
-      .deleteFrom('operation')
-      .where('actorDid', '=', req.actorDid)
-      .where('namespace', '=', req.namespace)
-      .returning('id')
-      .execute()
-    return new DeleteOperationsByActorAndNamespaceResponse({
-      deletedCount: deletedRows.length,
-    })
-  },
-})

package/src/routes/index.ts

@@ -1,28 +0,0 @@
-import { ConnectRouter } from '@connectrpc/connect'
-import { sql } from 'kysely'
-import { AppContext } from '../context.js'
-import { Service } from '../proto/bsync_connect.js'
-import addMuteOperation from './add-mute-operation.js'
-import addNotifOperation from './add-notif-operation.js'
-import deleteOperations from './delete-operations.js'
-import putOperation from './put-operation.js'
-import scanMuteOperations from './scan-mute-operations.js'
-import scanNotifOperations from './scan-notif-operations.js'
-import scanOperations from './scan-operations.js'
-
-export default (ctx: AppContext) => (router: ConnectRouter) => {
-  return router.service(Service, {
-    ...addMuteOperation(ctx),
-    ...scanMuteOperations(ctx),
-    ...addNotifOperation(ctx),
-    ...scanNotifOperations(ctx),
-    ...putOperation(ctx),
-    ...scanOperations(ctx),
-    ...deleteOperations(ctx),
-    async ping() {
-      const { db } = ctx
-      await sql`select 1`.execute(db.db)
-      return {}
-    },
-  })
-}

package/src/routes/put-operation.ts

@@ -1,115 +0,0 @@
-import { Code, ConnectError, ServiceImpl } from '@connectrpc/connect'
-import { sql } from 'kysely'
-import { ensureValidRecordKey } from '@atproto/syntax'
-import { AppContext } from '../context.js'
-import { Database } from '../db/index.js'
-import {
-  OperationMethod,
-  createOperationChannel,
-} from '../db/schema/operation.js'
-import { Service } from '../proto/bsync_connect.js'
-import {
-  Method,
-  PutOperationRequest,
-  PutOperationResponse,
-} from '../proto/bsync_pb.js'
-import { authWithApiKey } from './auth.js'
-import { isValidDid, validateNamespace } from './util.js'
-
-export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
-  async putOperation(req, handlerCtx) {
-    authWithApiKey(ctx, handlerCtx)
-    const { db } = ctx
-    const op = validateOp(req)
-    const id = await db.transaction(async (txn) => {
-      return putOp(txn, op)
-    })
-    return new PutOperationResponse({
-      operation: {
-        id: String(id),
-        actorDid: op.actorDid,
-        namespace: op.namespace,
-        key: op.key,
-        method: op.method,
-        payload: op.payload,
-      },
-    })
-  },
-})
-
-const putOp = async (db: Database, op: Operation) => {
-  const { ref } = db.db.dynamic
-  const { id } = await db.db
-    .insertInto('operation')
-    .values({
-      actorDid: op.actorDid,
-      namespace: op.namespace,
-      key: op.key,
-      method: op.method,
-      payload: op.payload,
-    })
-    .returning('id')
-    .executeTakeFirstOrThrow()
-  await sql`notify ${ref(createOperationChannel)}`.execute(db.db) // emitted transactionally
-  return id
-}
-
-const validateOp = (req: PutOperationRequest): Operation => {
-  try {
-    validateNamespace(req.namespace)
-  } catch (error) {
-    throw new ConnectError(
-      'operation namespace is invalid NSID',
-      Code.InvalidArgument,
-    )
-  }
-
-  if (!isValidDid(req.actorDid)) {
-    throw new ConnectError(
-      'operation actor_did is invalid DID',
-      Code.InvalidArgument,
-    )
-  }
-
-  try {
-    ensureValidRecordKey(req.key)
-  } catch (error) {
-    throw new ConnectError('operation key is required', Code.InvalidArgument)
-  }
-
-  if (
-    req.method !== Method.CREATE &&
-    req.method !== Method.UPDATE &&
-    req.method !== Method.DELETE
-  ) {
-    throw new ConnectError('operation method is invalid', Code.InvalidArgument)
-  }
-
-  if (req.method === Method.CREATE || req.method === Method.UPDATE) {
-    try {
-      JSON.parse(new TextDecoder().decode(req.payload))
-    } catch (error) {
-      throw new ConnectError(
-        'payload must be a valid JSON when method is CREATE or UPDATE',
-        Code.InvalidArgument,
-      )
-    }
-  }
-
-  if (req.method === Method.DELETE && req.payload.length > 0) {
-    throw new ConnectError(
-      'cannot specify a payload when method is DELETE',
-      Code.InvalidArgument,
-    )
-  }
-
-  return req as Operation
-}
-
-type Operation = {
-  actorDid: string
-  namespace: string
-  key: string
-  payload: Uint8Array<ArrayBuffer>
-  method: OperationMethod
-}

package/src/routes/scan-mute-operations.ts

@@ -1,65 +0,0 @@
-import { once } from 'node:events'
-import { ServiceImpl } from '@connectrpc/connect'
-import { AppContext } from '../context.js'
-import { createMuteOpChannel } from '../db/schema/mute_op.js'
-import { Service } from '../proto/bsync_connect.js'
-import { ScanMuteOperationsResponse } from '../proto/bsync_pb.js'
-import { authWithApiKey } from './auth.js'
-import { combineSignals, validCursor } from './util.js'
-
-export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
-  async scanMuteOperations(req, handlerCtx) {
-    authWithApiKey(ctx, handlerCtx)
-    const { db, events } = ctx
-    const limit = req.limit || 1000
-    const cursor = validCursor(req.cursor)
-    const nextMuteOpPromise = once(events, createMuteOpChannel, {
-      signal: combineSignals(
-        ctx.shutdown,
-        AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs),
-      ),
-    })
-    nextMuteOpPromise.catch(() => null) // ensure timeout is always handled
-
-    const nextMuteOpPageQb = db.db
-      .selectFrom('mute_op')
-      .selectAll()
-      .where('id', '>', cursor ?? -1)
-      .orderBy('id', 'asc')
-      .limit(limit)
-
-    let ops = await nextMuteOpPageQb.execute()
-
-    if (!ops.length) {
-      // if there were no ops on the page, wait for an event then try again.
-      try {
-        await nextMuteOpPromise
-      } catch (err) {
-        ctx.shutdown.throwIfAborted()
-        return new ScanMuteOperationsResponse({
-          operations: [],
-          cursor: req.cursor,
-        })
-      }
-      ops = await nextMuteOpPageQb.execute()
-      if (!ops.length) {
-        return new ScanMuteOperationsResponse({
-          operations: [],
-          cursor: req.cursor,
-        })
-      }
-    }
-
-    const lastOp = ops[ops.length - 1]
-
-    return new ScanMuteOperationsResponse({
-      operations: ops.map((op) => ({
-        id: op.id.toString(),
-        type: op.type,
-        actorDid: op.actorDid,
-        subject: op.subject,
-      })),
-      cursor: lastOp.id.toString(),
-    })
-  },
-})

package/src/routes/scan-notif-operations.ts

@@ -1,64 +0,0 @@
-import { once } from 'node:events'
-import { ServiceImpl } from '@connectrpc/connect'
-import { AppContext } from '../context.js'
-import { createNotifOpChannel } from '../db/schema/notif_op.js'
-import { Service } from '../proto/bsync_connect.js'
-import { ScanNotifOperationsResponse } from '../proto/bsync_pb.js'
-import { authWithApiKey } from './auth.js'
-import { combineSignals, validCursor } from './util.js'
-
-export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
-  async scanNotifOperations(req, handlerCtx) {
-    authWithApiKey(ctx, handlerCtx)
-    const { db, events } = ctx
-    const limit = req.limit || 1000
-    const cursor = validCursor(req.cursor)
-    const nextNotifOpPromise = once(events, createNotifOpChannel, {
-      signal: combineSignals(
-        ctx.shutdown,
-        AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs),
-      ),
-    })
-    nextNotifOpPromise.catch(() => null) // ensure timeout is always handled
-
-    const nextNotifOpPageQb = db.db
-      .selectFrom('notif_op')
-      .selectAll()
-      .where('id', '>', cursor ?? -1)
-      .orderBy('id', 'asc')
-      .limit(limit)
-
-    let ops = await nextNotifOpPageQb.execute()
-
-    if (!ops.length) {
-      // if there were no ops on the page, wait for an event then try again.
-      try {
-        await nextNotifOpPromise
-      } catch (err) {
-        ctx.shutdown.throwIfAborted()
-        return new ScanNotifOperationsResponse({
-          operations: [],
-          cursor: req.cursor,
-        })
-      }
-      ops = await nextNotifOpPageQb.execute()
-      if (!ops.length) {
-        return new ScanNotifOperationsResponse({
-          operations: [],
-          cursor: req.cursor,
-        })
-      }
-    }
-
-    const lastOp = ops[ops.length - 1]
-
-    return new ScanNotifOperationsResponse({
-      operations: ops.map((op) => ({
-        id: op.id.toString(),
-        actorDid: op.actorDid,
-        priority: op.priority ?? undefined,
-      })),
-      cursor: lastOp.id.toString(),
-    })
-  },
-})

package/src/routes/scan-operations.ts

@@ -1,67 +0,0 @@
-import { once } from 'node:events'
-import { ServiceImpl } from '@connectrpc/connect'
-import { AppContext } from '../context.js'
-import { createOperationChannel } from '../db/schema/operation.js'
-import { Service } from '../proto/bsync_connect.js'
-import { ScanOperationsResponse } from '../proto/bsync_pb.js'
-import { authWithApiKey } from './auth.js'
-import { combineSignals, validCursor } from './util.js'
-
-export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
-  async scanOperations(req, handlerCtx) {
-    authWithApiKey(ctx, handlerCtx)
-    const { db, events } = ctx
-    const limit = req.limit || 1000
-    const cursor = validCursor(req.cursor)
-    const nextOpPromise = once(events, createOperationChannel, {
-      signal: combineSignals(
-        ctx.shutdown,
-        AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs),
-      ),
-    })
-    nextOpPromise.catch(() => null) // ensure timeout is always handled
-
-    const nextOpPageQb = db.db
-      .selectFrom('operation')
-      .selectAll()
-      .where('id', '>', cursor ?? -1)
-      .orderBy('id', 'asc')
-      .limit(limit)
-
-    let ops = await nextOpPageQb.execute()
-
-    if (!ops.length) {
-      // if there were no ops on the page, wait for an event then try again.
-      try {
-        await nextOpPromise
-      } catch (err) {
-        ctx.shutdown.throwIfAborted()
-        return new ScanOperationsResponse({
-          operations: [],
-          cursor: req.cursor,
-        })
-      }
-      ops = await nextOpPageQb.execute()
-      if (!ops.length) {
-        return new ScanOperationsResponse({
-          operations: [],
-          cursor: req.cursor,
-        })
-      }
-    }
-
-    const lastOp = ops[ops.length - 1]
-
-    return new ScanOperationsResponse({
-      operations: ops.map((op) => ({
-        id: op.id.toString(),
-        actorDid: op.actorDid,
-        namespace: op.namespace,
-        key: op.key,
-        method: op.method,
-        payload: op.payload,
-      })),
-      cursor: lastOp.id.toString(),
-    })
-  },
-})

package/src/routes/util.ts

@@ -1,67 +0,0 @@
-import { Code, ConnectError } from '@connectrpc/connect'
-import {
-  InvalidDidError,
-  ensureValidAtUri,
-  ensureValidDid,
-  ensureValidNsid,
-} from '@atproto/syntax'
-
-export const validCursor = (cursor: string): number | null => {
-  if (cursor === '') return null
-  const int = parseInt(cursor, 10)
-  if (isNaN(int) || int < 0) {
-    throw new ConnectError('invalid cursor', Code.InvalidArgument)
-  }
-  return int
-}
-
-export const combineSignals = (a: AbortSignal, b: AbortSignal): AbortSignal => {
-  const controller = new AbortController()
-  for (const signal of [a, b]) {
-    if (signal.aborted) {
-      controller.abort()
-      return signal
-    }
-    signal.addEventListener('abort', () => controller.abort(signal.reason), {
-      // @ts-ignore https://github.com/DefinitelyTyped/DefinitelyTyped/pull/68625
-      signal: controller.signal,
-    })
-  }
-  return controller.signal
-}
-
-export const isValidDid = (did: string) => {
-  try {
-    ensureValidDid(did)
-    return true
-  } catch (err) {
-    if (err instanceof InvalidDidError) {
-      return false
-    }
-    throw err
-  }
-}
-
-export const isValidAtUri = (uri: string) => {
-  try {
-    ensureValidAtUri(uri)
-    return true
-  } catch {
-    return false
-  }
-}
-
-export const validateNamespace = (namespace: string): void => {
-  const parts = namespace.split('#')
-
-  if (parts.length !== 1 && parts.length !== 2) {
-    throw new Error('namespace must be in the format "nsid[#fragment]"')
-  }
-
-  const [nsid, fragment] = parts
-
-  ensureValidNsid(nsid)
-  if (fragment && !/^[a-zA-Z][a-zA-Z0-9]*$/.test(fragment)) {
-    throw new Error('namespace fragment must be a valid identifier')
-  }
-}

package/tests/delete-operations.test.ts

@@ -1,108 +0,0 @@
-import getPort from 'get-port'
-import {
-  BsyncClient,
-  BsyncService,
-  Database,
-  authWithApiKey,
-  createClient,
-  envToCfg,
-} from '../src/index.js'
-import { Method } from '../src/proto/bsync_pb.js'
-
-describe('operations', () => {
-  let bsync: BsyncService
-  let client: BsyncClient
-
-  const validPayload0 = Buffer.from(JSON.stringify({ value: 0 }))
-  const validPayload1 = Buffer.from(JSON.stringify({ value: 1 }))
-
-  beforeAll(async () => {
-    bsync = await BsyncService.create(
-      envToCfg({
-        port: await getPort(),
-        dbUrl: process.env.DB_POSTGRES_URL,
-        dbSchema: 'bsync_delete_operations',
-        apiKeys: ['key-1'],
-        longPollTimeoutMs: 500,
-      }),
-    )
-    await bsync.ctx.db.migrateToLatestOrThrow()
-    await bsync.start()
-    client = createClient({
-      httpVersion: '1.1',
-      baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`,
-      interceptors: [authWithApiKey('key-1')],
-    })
-  })
-
-  afterAll(async () => {
-    await bsync.destroy()
-  })
-
-  beforeEach(async () => {
-    await clearOps(bsync.ctx.db)
-  })
-
-  it('deletes', async () => {
-    const res1 = await client.putOperation({
-      actorDid: 'did:example:a',
-      namespace: 'app.bsky.some.col',
-      key: 'key1',
-      method: Method.CREATE,
-      payload: validPayload0,
-    })
-    const res2 = await client.putOperation({
-      actorDid: 'did:example:a',
-      namespace: 'app.bsky.other.col#id',
-      key: 'key1',
-      method: Method.UPDATE,
-      payload: validPayload1,
-    })
-
-    expect(res1.operation?.id).toBe('1')
-    expect(res2.operation?.id).toBe('2')
-    expect(await dumpOps(bsync.ctx.db)).toStrictEqual([
-      {
-        id: 1,
-        actorDid: 'did:example:a',
-        namespace: 'app.bsky.some.col',
-        key: 'key1',
-        method: Method.CREATE,
-        payload: validPayload0,
-        createdAt: expect.any(Date),
-      },
-      {
-        id: 2,
-        actorDid: 'did:example:a',
-        namespace: 'app.bsky.other.col#id',
-        key: 'key1',
-        method: Method.UPDATE,
-        payload: validPayload1,
-        createdAt: expect.any(Date),
-      },
-    ])
-
-    await client.deleteOperationsByActorAndNamespace({
-      actorDid: 'did:example:a',
-      namespace: 'app.bsky.some.col',
-    })
-    await client.deleteOperationsByActorAndNamespace({
-      actorDid: 'did:example:a',
-      namespace: 'app.bsky.other.col#id',
-    })
-
-    expect(await dumpOps(bsync.ctx.db)).toStrictEqual([])
-  })
-})
-
-const dumpOps = async (db: Database) => {
-  return db.db
-    .selectFrom('operation')
-    .selectAll()
-    .orderBy('id', 'asc')
-    .execute()
-}
-
-const clearOps = async (db: Database) => {
-  await db.db.deleteFrom('operation').execute()
-}