@atproto/bsync 0.0.19 → 0.0.21

package/src/routes/put-operation.ts

@@ -0,0 +1,127 @@
+import { Code, ConnectError, ServiceImpl } from '@connectrpc/connect'
+import { sql } from 'kysely'
+import { ensureValidNsid, ensureValidRecordKey } from '@atproto/syntax'
+import { AppContext } from '../context'
+import { Database } from '../db'
+import { OperationMethod, createOperationChannel } from '../db/schema/operation'
+import { Service } from '../proto/bsync_connect'
+import {
+  Method,
+  PutOperationRequest,
+  PutOperationResponse,
+} from '../proto/bsync_pb'
+import { authWithApiKey } from './auth'
+import { isValidDid } from './util'
+
+export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
+  async putOperation(req, handlerCtx) {
+    authWithApiKey(ctx, handlerCtx)
+    const { db } = ctx
+    const op = validateOp(req)
+    const id = await db.transaction(async (txn) => {
+      return putOp(txn, op)
+    })
+    return new PutOperationResponse({
+      operation: {
+        id: String(id),
+        actorDid: op.actorDid,
+        namespace: op.namespace,
+        key: op.key,
+        method: op.method,
+        payload: op.payload,
+      },
+    })
+  },
+})
+
+const putOp = async (db: Database, op: Operation) => {
+  const { ref } = db.db.dynamic
+  const { id } = await db.db
+    .insertInto('operation')
+    .values({
+      actorDid: op.actorDid,
+      namespace: op.namespace,
+      key: op.key,
+      method: op.method,
+      payload: op.payload,
+    })
+    .returning('id')
+    .executeTakeFirstOrThrow()
+  await sql`notify ${ref(createOperationChannel)}`.execute(db.db) // emitted transactionally
+  return id
+}
+
+const validateOp = (req: PutOperationRequest): Operation => {
+  try {
+    validateNamespace(req.namespace)
+  } catch (error) {
+    throw new ConnectError(
+      'operation namespace is invalid NSID',
+      Code.InvalidArgument,
+    )
+  }
+
+  if (!isValidDid(req.actorDid)) {
+    throw new ConnectError(
+      'operation actor_did is invalid DID',
+      Code.InvalidArgument,
+    )
+  }
+
+  try {
+    ensureValidRecordKey(req.key)
+  } catch (error) {
+    throw new ConnectError('operation key is required', Code.InvalidArgument)
+  }
+
+  if (
+    req.method !== Method.CREATE &&
+    req.method !== Method.UPDATE &&
+    req.method !== Method.DELETE
+  ) {
+    throw new ConnectError('operation method is invalid', Code.InvalidArgument)
+  }
+
+  if (req.method === Method.CREATE || req.method === Method.UPDATE) {
+    try {
+      JSON.parse(new TextDecoder().decode(req.payload))
+    } catch (error) {
+      throw new ConnectError(
+        'payload must be a valid JSON when method is CREATE or UPDATE',
+        Code.InvalidArgument,
+      )
+    }
+  }
+
+  if (req.method === Method.DELETE && req.payload.length > 0) {
+    throw new ConnectError(
+      'cannot specify a payload when method is DELETE',
+      Code.InvalidArgument,
+    )
+  }
+
+  return req as Operation
+}
+
+const validateNamespace = (namespace: string): void => {
+  const parts = namespace.split('#')
+
+  if (parts.length !== 1 && parts.length !== 2) {
+    throw new Error('namespace must be in the format "nsid[#fragment]"')
+  }
+
+  const [nsid, fragment] = parts
+
+  ensureValidNsid(nsid)
+  if (fragment && !/^[a-zA-Z][a-zA-Z0-9]*$/.test(fragment)) {
+    throw new Error('namespace fragment must be a valid identifier')
+  }
+}
+
+type Operation = {
+  actorDid: string
+  namespace: string
+  key: string
+  payload: Uint8Array
+  method: OperationMethod
+}

package/src/routes/scan-operations.ts

@@ -0,0 +1,67 @@
+import { once } from 'node:events'
+import { ServiceImpl } from '@connectrpc/connect'
+import { AppContext } from '../context'
+import { createOperationChannel } from '../db/schema/operation'
+import { Service } from '../proto/bsync_connect'
+import { ScanOperationsResponse } from '../proto/bsync_pb'
+import { authWithApiKey } from './auth'
+import { combineSignals, validCursor } from './util'
+
+export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
+  async scanOperations(req, handlerCtx) {
+    authWithApiKey(ctx, handlerCtx)
+    const { db, events } = ctx
+    const limit = req.limit || 1000
+    const cursor = validCursor(req.cursor)
+    const nextOpPromise = once(events, createOperationChannel, {
+      signal: combineSignals(
+        ctx.shutdown,
+        AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs),
+      ),
+    })
+    nextOpPromise.catch(() => null) // ensure timeout is always handled
+
+    const nextOpPageQb = db.db
+      .selectFrom('operation')
+      .selectAll()
+      .where('id', '>', cursor ?? -1)
+      .orderBy('id', 'asc')
+      .limit(limit)
+
+    let ops = await nextOpPageQb.execute()
+
+    if (!ops.length) {
+      // if there were no ops on the page, wait for an event then try again.
+      try {
+        await nextOpPromise
+      } catch (err) {
+        ctx.shutdown.throwIfAborted()
+        return new ScanOperationsResponse({
+          operations: [],
+          cursor: req.cursor,
+        })
+      }
+      ops = await nextOpPageQb.execute()
+      if (!ops.length) {
+        return new ScanOperationsResponse({
+          operations: [],
+          cursor: req.cursor,
+        })
+      }
+    }
+
+    const lastOp = ops[ops.length - 1]
+
+    return new ScanOperationsResponse({
+      operations: ops.map((op) => ({
+        id: op.id.toString(),
+        actorDid: op.actorDid,
+        namespace: op.namespace,
+        key: op.key,
+        method: op.method,
+        payload: op.payload,
+      })),
+      cursor: lastOp.id.toString(),
+    })
+  },
+})

package/tests/operations.test.ts

@@ -0,0 +1,327 @@
+import assert from 'node:assert'
+import { Code, ConnectError } from '@connectrpc/connect'
+import getPort from 'get-port'
+import { wait } from '@atproto/common'
+import {
+  BsyncClient,
+  BsyncService,
+  Database,
+  authWithApiKey,
+  createClient,
+  envToCfg,
+} from '../src'
+import { Method, Operation } from '../src/proto/bsync_pb'
+
+describe('operations', () => {
+  let bsync: BsyncService
+  let client: BsyncClient
+
+  const validPayload0 = Buffer.from(JSON.stringify({ value: 0 }))
+  const validPayload1 = Buffer.from(JSON.stringify({ value: 1 }))
+  const invalidPayload = Buffer.from('{invalid json}')
+
+  beforeAll(async () => {
+    bsync = await BsyncService.create(
+      envToCfg({
+        port: await getPort(),
+        dbUrl: process.env.DB_POSTGRES_URL,
+        dbSchema: 'bsync_operations',
+        apiKeys: ['key-1'],
+        longPollTimeoutMs: 500,
+      }),
+    )
+    await bsync.ctx.db.migrateToLatestOrThrow()
+    await bsync.start()
+    client = createClient({
+      httpVersion: '1.1',
+      baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`,
+      interceptors: [authWithApiKey('key-1')],
+    })
+  })
+
+  afterAll(async () => {
+    await bsync.destroy()
+  })
+
+  beforeEach(async () => {
+    await clearOps(bsync.ctx.db)
+  })
+
+  describe('putOperation', () => {
+    it('requires auth.', async () => {
+      // unauthed
+      const unauthedClient = createClient({
+        httpVersion: '1.1',
+        baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`,
+      })
+      const tryPutOperation1 = unauthedClient.putOperation({
+        actorDid: 'did:example:a',
+        namespace: 'app.bsky.some.col',
+        key: 'key1',
+        method: Method.CREATE,
+        payload: validPayload0,
+      })
+      await expect(tryPutOperation1).rejects.toEqual(
+        new ConnectError('missing auth', Code.Unauthenticated),
+      )
+      // bad auth
+      const badauthedClient = createClient({
+        httpVersion: '1.1',
+        baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`,
+        interceptors: [authWithApiKey('key-bad')],
+      })
+      const tryPutOperation2 = badauthedClient.putOperation({
+        actorDid: 'did:example:a',
+        namespace: 'app.bsky.some.col',
+        key: 'key1',
+        method: Method.CREATE,
+        payload: validPayload0,
+      })
+      await expect(tryPutOperation2).rejects.toEqual(
+        new ConnectError('invalid api key', Code.Unauthenticated),
+      )
+    })
+
+    it('fails on bad inputs.', async () => {
+      await expect(
+        client.putOperation({
+          actorDid: 'did:example:a',
+          namespace: 'bad-namespace',
+          key: 'key1',
+          method: Method.CREATE,
+          payload: validPayload0,
+        }),
+      ).rejects.toEqual(
+        new ConnectError(
+          'operation namespace is invalid NSID',
+          Code.InvalidArgument,
+        ),
+      )
+      await expect(
+        client.putOperation({
+          actorDid: 'bad-did',
+          namespace: 'app.bsky.some.col',
+          key: 'key1',
+          method: Method.CREATE,
+          payload: validPayload0,
+        }),
+      ).rejects.toEqual(
+        new ConnectError(
+          'operation actor_did is invalid DID',
+          Code.InvalidArgument,
+        ),
+      )
+      await expect(
+        client.putOperation({
+          actorDid: 'did:example:a',
+          namespace: 'app.bsky.some.col',
+          key: '',
+          method: Method.CREATE,
+          payload: validPayload0,
+        }),
+      ).rejects.toEqual(
+        new ConnectError('operation key is required', Code.InvalidArgument),
+      )
+      await expect(
+        client.putOperation({
+          actorDid: 'did:example:a',
+          namespace: 'app.bsky.some.col',
+          key: 'key1',
+          method: Method.UNSPECIFIED,
+          payload: validPayload0,
+        }),
+      ).rejects.toEqual(
+        new ConnectError('operation method is invalid', Code.InvalidArgument),
+      )
+      await expect(
+        client.putOperation({
+          actorDid: 'did:example:a',
+          namespace: 'app.bsky.some.col',
+          key: 'key1',
+          method: Method.CREATE,
+          payload: invalidPayload,
+        }),
+      ).rejects.toEqual(
+        new ConnectError(
+          'payload must be a valid JSON when method is CREATE or UPDATE',
+          Code.InvalidArgument,
+        ),
+      )
+      await expect(
+        client.putOperation({
+          actorDid: 'did:example:a',
+          namespace: 'app.bsky.some.col',
+          key: 'key1',
+          method: Method.UPDATE,
+          payload: invalidPayload,
+        }),
+      ).rejects.toEqual(
+        new ConnectError(
+          'payload must be a valid JSON when method is CREATE or UPDATE',
+          Code.InvalidArgument,
+        ),
+      )
+      await expect(
+        client.putOperation({
+          actorDid: 'did:example:a',
+          namespace: 'app.bsky.some.col',
+          key: 'key1',
+          method: Method.DELETE,
+          payload: validPayload0,
+        }),
+      ).rejects.toEqual(
+        new ConnectError(
+          'cannot specify a payload when method is DELETE',
+          Code.InvalidArgument,
+        ),
+      )
+    })
+
+    it('puts operations.', async () => {
+      const res1 = await client.putOperation({
+        actorDid: 'did:example:a',
+        namespace: 'app.bsky.some.col',
+        key: 'key1',
+        method: Method.CREATE,
+        payload: validPayload0,
+      })
+      const res2 = await client.putOperation({
+        actorDid: 'did:example:a',
+        namespace: 'app.bsky.other.col#id',
+        key: 'key1',
+        method: Method.UPDATE,
+        payload: validPayload1,
+      })
+
+      expect(res1.operation?.id).toBe('1')
+      expect(res2.operation?.id).toBe('2')
+      expect(await dumpOps(bsync.ctx.db)).toStrictEqual([
+        {
+          id: 1,
+          actorDid: 'did:example:a',
+          namespace: 'app.bsky.some.col',
+          key: 'key1',
+          method: Method.CREATE,
+          payload: validPayload0,
+          createdAt: expect.any(Date),
+        },
+        {
+          id: 2,
+          actorDid: 'did:example:a',
+          namespace: 'app.bsky.other.col#id',
+          key: 'key1',
+          method: Method.UPDATE,
+          payload: validPayload1,
+          createdAt: expect.any(Date),
+        },
+      ])
+    })
+
+    it('returns the operations on creation.', async () => {
+      const res = await client.putOperation({
+        actorDid: 'did:example:a',
+        namespace: 'app.bsky.some.col',
+        key: 'key1',
+        method: Method.CREATE,
+        payload: validPayload0,
+      })
+
+      const op = res.operation
+      assert(op)
+      // Compare each field individually to avoid custom serialization by proto response objects.
+      expect(op.id).toBe('3')
+      expect(op.actorDid).toBe('did:example:a')
+      expect(op.namespace).toBe('app.bsky.some.col')
+      expect(op.key).toBe('key1')
+      expect(op.method).toBe(Method.CREATE)
+      expect(op.payload).toEqual(new Uint8Array(validPayload0))
+    })
+  })
+
+  describe('scanOperations', () => {
+    it('requires auth.', async () => {
+      // unauthed
+      const unauthedClient = createClient({
+        httpVersion: '1.1',
+        baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`,
+      })
+      const tryScanOperations1 = unauthedClient.scanOperations({})
+      await expect(tryScanOperations1).rejects.toEqual(
+        new ConnectError('missing auth', Code.Unauthenticated),
+      )
+      // bad auth
+      const badauthedClient = createClient({
+        httpVersion: '1.1',
+        baseUrl: `http://localhost:${bsync.ctx.cfg.service.port}`,
+        interceptors: [authWithApiKey('key-bad')],
+      })
+      const tryScanOperations2 = badauthedClient.scanOperations({})
+      await expect(tryScanOperations2).rejects.toEqual(
+        new ConnectError('invalid api key', Code.Unauthenticated),
+      )
+    })
+
+    it('pages over created ops.', async () => {
+      // add 100 ops
+      for (let i = 0; i < 100; ++i) {
+        await client.putOperation({
+          actorDid: `did:example:${i}`,
+          namespace: 'app.bsky.some.col',
+          key: 'key1',
+          method: Method.CREATE,
+          payload: validPayload0,
+        })
+      }
+
+      let cursor: string | undefined
+      const operations: Operation[] = []
+      do {
+        const res = await client.scanOperations({
+          cursor,
+          limit: 30,
+        })
+        operations.push(...res.operations)
+        cursor = res.operations.length ? res.cursor : undefined
+      } while (cursor)
+
+      expect(operations.length).toEqual(100)
+      const operationIds = operations.map((op) => parseInt(op.id, 10))
+      const ascending = (a: number, b: number) => a - b
+      expect(operationIds).toEqual([...operationIds].sort(ascending))
+    })
+
+    it('supports long-poll, finding an operation.', async () => {
+      const scanPromise = client.scanOperations({})
+      await wait(100) // would be complete by now if it wasn't long-polling for an item
+      const { operation } = await client.putOperation({
+        actorDid: 'did:example:a',
+        namespace: 'app.bsky.some.col',
+        key: 'key1',
+        method: Method.CREATE,
+        payload: validPayload0,
+      })
+      const res = await scanPromise
+      expect(res.operations.length).toEqual(1)
+      expect(res.operations[0]).toEqual(operation)
+      expect(res.cursor).toEqual(operation?.id)
+    })
+
+    it('supports long-poll, not finding an operation.', async () => {
+      const res = await client.scanOperations({})
+      expect(res.cursor).toEqual('')
+      expect(res.operations).toEqual([])
+    })
+  })
+})
+
+const dumpOps = async (db: Database) => {
+  return db.db
+    .selectFrom('operation')
+    .selectAll()
+    .orderBy('id', 'asc')
+    .execute()
+}
+
+const clearOps = async (db: Database) => {
+  await db.db.deleteFrom('operation').execute()
+}

package/tsconfig.build.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/client.ts","./src/config.ts","./src/context.ts","./src/index.ts","./src/logger.ts","./src/db/index.ts","./src/db/types.ts","./src/db/migrations/20240108T220751294Z-init.ts","./src/db/migrations/20240717T224303472Z-notif-ops.ts","./src/db/migrations/index.ts","./src/db/migrations/provider.ts","./src/db/schema/index.ts","./src/db/schema/mute_item.ts","./src/db/schema/mute_op.ts","./src/db/schema/notif_item.ts","./src/db/schema/notif_op.ts","./src/proto/bsync_connect.ts","./src/proto/bsync_pb.ts","./src/routes/add-mute-operation.ts","./src/routes/add-notif-operation.ts","./src/routes/auth.ts","./src/routes/index.ts","./src/routes/scan-mute-operations.ts","./src/routes/scan-notif-operations.ts","./src/routes/util.ts"],"version":"5.8.2"}
+{"root":["./src/client.ts","./src/config.ts","./src/context.ts","./src/index.ts","./src/logger.ts","./src/db/index.ts","./src/db/types.ts","./src/db/migrations/20240108T220751294Z-init.ts","./src/db/migrations/20240717T224303472Z-notif-ops.ts","./src/db/migrations/20250527T022203400Z-add-operation.ts","./src/db/migrations/20250603T163446567Z-alter-operation.ts","./src/db/migrations/index.ts","./src/db/migrations/provider.ts","./src/db/schema/index.ts","./src/db/schema/mute_item.ts","./src/db/schema/mute_op.ts","./src/db/schema/notif_item.ts","./src/db/schema/notif_op.ts","./src/db/schema/operation.ts","./src/proto/bsync_connect.ts","./src/proto/bsync_pb.ts","./src/routes/add-mute-operation.ts","./src/routes/add-notif-operation.ts","./src/routes/auth.ts","./src/routes/index.ts","./src/routes/put-operation.ts","./src/routes/scan-mute-operations.ts","./src/routes/scan-notif-operations.ts","./src/routes/scan-operations.ts","./src/routes/util.ts"],"version":"5.8.2"}

package/tsconfig.tests.tsbuildinfo

@@ -1 +1 @@
-{"root":["./tests/mutes.test.ts","./tests/notifications.test.ts"],"version":"5.8.2"}
+{"root":["./tests/mutes.test.ts","./tests/notifications.test.ts","./tests/operations.test.ts"],"version":"5.8.3"}