@atproto/ozone 0.0.12 → 0.0.13

package/tests/moderation.test.ts

@@ -607,7 +607,7 @@ describe('moderation', () => {
         },
         {
           encoding: 'application/json',
-          headers: network.bsky.adminAuthHeaders('triage'),
+          headers: network.ozone.adminAuthHeaders('triage'),
         },
       )
       await expect(attemptLabel).rejects.toThrow(
@@ -748,7 +748,7 @@ describe('moderation', () => {
         },
         {
           encoding: 'application/json',
-          headers: network.bsky.adminAuthHeaders('moderator'),
+          headers: network.ozone.adminAuthHeaders('moderator'),
         },
       )
       // cleanup
@@ -775,11 +775,11 @@ describe('moderation', () => {
           },
           {
             encoding: 'application/json',
-            headers: network.bsky.adminAuthHeaders('triage'),
+            headers: network.ozone.adminAuthHeaders('triage'),
           },
         )
       await expect(attemptTakedownTriage).rejects.toThrow(
-        'Must be a full moderator to perform an account takedown',
+        'Must be a full moderator to take this type of action',
       )
     })
 
@@ -800,7 +800,7 @@ describe('moderation', () => {
       const { data: statusesAfterTakedown } =
         await agent.api.com.atproto.admin.queryModerationStatuses(
           { subject: sc.dids.bob },
-          { headers: network.bsky.adminAuthHeaders('moderator') },
+          { headers: network.ozone.adminAuthHeaders('moderator') },
         )
 
       expect(statusesAfterTakedown.subjectStatuses[0]).toMatchObject({
@@ -818,11 +818,11 @@ describe('moderation', () => {
       const [{ data: eventList }, { data: statuses }] = await Promise.all([
         agent.api.com.atproto.admin.queryModerationEvents(
           { subject: sc.dids.bob },
-          { headers: network.bsky.adminAuthHeaders('moderator') },
+          { headers: network.ozone.adminAuthHeaders('moderator') },
         ),
         agent.api.com.atproto.admin.queryModerationStatuses(
           { subject: sc.dids.bob },
-          { headers: network.bsky.adminAuthHeaders('moderator') },
+          { headers: network.ozone.adminAuthHeaders('moderator') },
         ),
       ])
 
@@ -879,7 +879,7 @@ describe('moderation', () => {
         },
         {
           encoding: 'application/json',
-          headers: network.bsky.adminAuthHeaders(),
+          headers: network.ozone.adminAuthHeaders(),
         },
       )
       return result.data
@@ -901,7 +901,7 @@ describe('moderation', () => {
         },
         {
           encoding: 'application/json',
-          headers: network.bsky.adminAuthHeaders(),
+          headers: network.ozone.adminAuthHeaders(),
         },
       )
     }
@@ -909,7 +909,7 @@ describe('moderation', () => {
     async function getRecordLabels(uri: string) {
       const result = await agent.api.com.atproto.admin.getRecord(
         { uri },
-        { headers: network.bsky.adminAuthHeaders() },
+        { headers: network.ozone.adminAuthHeaders() },
       )
       const labels = result.data.labels ?? []
       return labels.map((l) => l.val)
@@ -918,7 +918,7 @@ describe('moderation', () => {
     async function getRepoLabels(did: string) {
       const result = await agent.api.com.atproto.admin.getRepo(
         { did },
-        { headers: network.bsky.adminAuthHeaders() },
+        { headers: network.ozone.adminAuthHeaders() },
       )
       const labels = result.data.labels ?? []
       return labels.map((l) => l.val)
@@ -933,7 +933,7 @@ describe('moderation', () => {
       const { ctx } = network.bsky
       post = sc.posts[sc.dids.carol][0]
       blob = post.images[1]
-      imageUri = ctx.imgUriBuilder
+      imageUri = ctx.views.imgUriBuilder
         .getPresetUri(
           'feed_thumbnail',
           sc.dids.carol,
@@ -974,7 +974,8 @@ describe('moderation', () => {
       })
     })
 
-    it('prevents image blob from being served, even when cached.', async () => {
+    // @TODO add back in with image invalidation, see bluesky-social/atproto#2087
+    it.skip('prevents image blob from being served, even when cached.', async () => {
       const fetchImage = await fetch(imageUri)
       expect(fetchImage.status).toEqual(404)
       expect(await fetchImage.json()).toEqual({ message: 'Image not found' })

package/tests/query-labels.test.ts

@@ -0,0 +1,163 @@
+import AtpAgent from '@atproto/api'
+import { TestNetwork } from '@atproto/dev-env'
+import { DisconnectError, Subscription } from '@atproto/xrpc-server'
+import { ids, lexicons } from '../src/lexicon/lexicons'
+import { Label } from '../src/lexicon/types/com/atproto/label/defs'
+import {
+  OutputSchema as LabelMessage,
+  isLabels,
+} from '../src/lexicon/types/com/atproto/label/subscribeLabels'
+
+describe('ozone query labels', () => {
+  let network: TestNetwork
+  let agent: AtpAgent
+
+  let labels: Label[]
+
+  beforeAll(async () => {
+    network = await TestNetwork.create({
+      dbPostgresSchema: 'ozone_query_labels',
+    })
+
+    agent = network.ozone.getClient()
+
+    labels = [
+      {
+        src: 'did:example:labeler',
+        uri: 'did:example:blah',
+        val: 'spam',
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+      {
+        src: 'did:example:labeler',
+        uri: 'did:example:blah',
+        val: 'impersonation',
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+      {
+        src: 'did:example:labeler',
+        uri: 'at://did:example:blah/app.bsky.feed.post/1234abcde',
+        val: 'spam',
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+      {
+        src: 'did:example:labeler',
+        uri: 'at://did:example:blah/app.bsky.feed.post/1234abcfg',
+        val: 'spam',
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+      {
+        src: 'did:example:labeler',
+        uri: 'at://did:example:blah/app.bsky.actor.profile/self',
+        val: 'spam',
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+      {
+        src: 'did:example:labeler',
+        uri: 'did:example:thing',
+        val: 'spam',
+        neg: false,
+        cts: new Date().toISOString(),
+      },
+    ]
+
+    const modService = network.ozone.ctx.modService(network.ozone.ctx.db)
+    await modService.createLabels(labels)
+  })
+
+  afterAll(async () => {
+    await network.close()
+  })
+
+  it('returns all labels', async () => {
+    const res = await agent.api.com.atproto.label.queryLabels({
+      uriPatterns: ['*'],
+    })
+    expect(res.data.labels).toEqual(labels)
+  })
+
+  it('returns all labels even when an additional pattern is supplied', async () => {
+    const res = await agent.api.com.atproto.label.queryLabels({
+      uriPatterns: ['*', 'did:example:blah'],
+    })
+    expect(res.data.labels).toEqual(labels)
+  })
+
+  it('returns all labels that match an exact uri pattern', async () => {
+    const res = await agent.api.com.atproto.label.queryLabels({
+      uriPatterns: ['did:example:blah'],
+    })
+    expect(res.data.labels).toEqual(labels.slice(0, 2))
+  })
+
+  it('returns all labels that match one of multiple exact uris', async () => {
+    const res = await agent.api.com.atproto.label.queryLabels({
+      uriPatterns: [
+        'at://did:example:blah/app.bsky.feed.post/1234abcfg',
+        'at://did:example:blah/app.bsky.actor.profile/self',
+      ],
+    })
+    expect(res.data.labels).toEqual(labels.slice(3, 5))
+  })
+
+  it('returns all labels that match one of multiple uris, exact & glob', async () => {
+    const res = await agent.api.com.atproto.label.queryLabels({
+      uriPatterns: ['at://did:example:blah/app.bsky*', 'did:example:blah'],
+    })
+    expect(res.data.labels).toEqual(labels.slice(0, 5))
+  })
+
+  it('paginates', async () => {
+    const res1 = await agent.api.com.atproto.label.queryLabels({
+      uriPatterns: ['at://did:example:blah/app.bsky*', 'did:example:blah'],
+      limit: 3,
+    })
+    const res2 = await agent.api.com.atproto.label.queryLabels({
+      uriPatterns: ['at://did:example:blah/app.bsky*', 'did:example:blah'],
+      limit: 3,
+      cursor: res1.data.cursor,
+    })
+
+    expect([...res1.data.labels, ...res2.data.labels]).toEqual(
+      labels.slice(0, 5),
+    )
+  })
+
+  describe('subscribeLabels', () => {
+    it('streams all labels from initial cursor.', async () => {
+      const ac = new AbortController()
+      let doneTimer: NodeJS.Timeout
+      const resetDoneTimer = () => {
+        clearTimeout(doneTimer)
+        doneTimer = setTimeout(() => ac.abort(new DisconnectError()), 100)
+      }
+      const sub = new Subscription({
+        signal: ac.signal,
+        service: agent.service.origin.replace('http://', 'ws://'),
+        method: ids.ComAtprotoLabelSubscribeLabels,
+        getParams() {
+          return { cursor: 0 }
+        },
+        validate(obj) {
+          return lexicons.assertValidXrpcMessage<LabelMessage>(
+            ids.ComAtprotoLabelSubscribeLabels,
+            obj,
+          )
+        },
+      })
+      const streamedLabels: Label[] = []
+      for await (const message of sub) {
+        resetDoneTimer()
+        if (isLabels(message)) {
+          streamedLabels.push(...message.labels)
+        }
+      }
+      expect(streamedLabels).toEqual(labels)
+    })
+  })
+})

package/tests/repo-search.test.ts

@@ -117,7 +117,6 @@ describe('admin repo search view', () => {
       { headers },
     )
 
-    expect(full.data.repos.length).toEqual(15)
     expect(results(paginatedAll)).toEqual(results([full.data]))
   })
 })

package/tests/sequencer.test.ts

@@ -0,0 +1,222 @@
+import { TestNetwork } from '@atproto/dev-env'
+import { readFromGenerator, wait } from '@atproto/common'
+import { LabelsEvt, Sequencer } from '../src/sequencer'
+import Outbox from '../src/sequencer/outbox'
+import { randomStr } from '@atproto/crypto'
+import { Label } from '../src/lexicon/types/com/atproto/label/defs'
+
+describe('sequencer', () => {
+  let network: TestNetwork
+  let sequencer: Sequencer
+
+  let totalEvts = 0
+  let lastSeen: number
+
+  beforeAll(async () => {
+    network = await TestNetwork.create({
+      dbPostgresSchema: 'ozone_sequencer',
+    })
+    sequencer = network.ozone.ctx.sequencer
+  })
+
+  afterAll(async () => {
+    await network.close()
+  })
+
+  const loadFromDb = (lastSeen: number) => {
+    return sequencer.db.db
+      .selectFrom('label')
+      .selectAll()
+      .where('id', '>', lastSeen)
+      .orderBy('id', 'asc')
+      .execute()
+  }
+
+  const evtToDbRow = (e: LabelsEvt) => {
+    const label = e.labels[0]
+    return {
+      id: e.seq,
+      ...label,
+      cid: label.cid ? label.cid : '',
+    }
+  }
+
+  const caughtUp = (outbox: Outbox): (() => Promise<boolean>) => {
+    return async () => {
+      const lastEvt = await outbox.sequencer.curr()
+      if (lastEvt === null) return true
+      return outbox.lastSeen >= (lastEvt ?? 0)
+    }
+  }
+
+  const createLabels = async (count: number): Promise<Label[]> => {
+    const labels: Label[] = []
+    for (let i = 0; i < count; i++) {
+      const did = `did:example:${randomStr(10, 'base32')}`
+      const label = {
+        src: 'did:example:labeler',
+        uri: did,
+        val: 'spam',
+        neg: false,
+        cts: new Date().toISOString(),
+      }
+      await network.ozone.ctx.db.transaction((dbTxn) =>
+        network.ozone.ctx.modService(dbTxn).createLabels([label]),
+      )
+      labels.push(label)
+    }
+    return labels
+  }
+
+  it('sends to outbox', async () => {
+    const count = 20
+    totalEvts += count
+    await createLabels(count)
+    const outbox = new Outbox(sequencer)
+    const evts = await readFromGenerator(outbox.events(-1), caughtUp(outbox))
+    expect(evts.length).toBe(totalEvts)
+
+    const fromDb = await loadFromDb(-1)
+    expect(evts.map(evtToDbRow)).toEqual(fromDb)
+
+    lastSeen = evts.at(-1)?.seq ?? lastSeen
+  })
+
+  it('sequences negative labels', async () => {
+    const count = 5
+    totalEvts += count
+    const created = await createLabels(count)
+    const toNegate = created
+      .slice(0, 2)
+      .map((l) => ({ ...l, neg: true, cts: new Date().toISOString() }))
+    await network.ozone.ctx
+      .modService(network.ozone.ctx.db)
+      .createLabels(toNegate)
+
+    const outbox = new Outbox(sequencer)
+    const evts = await readFromGenerator(
+      outbox.events(lastSeen),
+      caughtUp(outbox),
+    )
+    expect(evts.length).toBe(count)
+
+    const fromDb = await loadFromDb(lastSeen)
+    expect(evts.map(evtToDbRow)).toEqual(fromDb)
+    expect(evts[3].labels[0].uri).toEqual(toNegate[0].uri)
+    expect(evts[3].labels[0].neg).toBe(true)
+    expect(evts[4].labels[0].uri).toEqual(toNegate[1].uri)
+    expect(evts[4].labels[0].neg).toBe(true)
+
+    lastSeen = evts.at(-1)?.seq ?? lastSeen
+  })
+
+  it('handles cut over', async () => {
+    const count = 20
+    totalEvts += count
+    const outbox = new Outbox(sequencer)
+    const createPromise = createLabels(count)
+    const [evts] = await Promise.all([
+      readFromGenerator(outbox.events(-1), caughtUp(outbox), createPromise),
+      createPromise,
+    ])
+    expect(evts.length).toBe(totalEvts)
+
+    const fromDb = await loadFromDb(-1)
+    expect(evts.map(evtToDbRow)).toEqual(fromDb)
+
+    lastSeen = evts.at(-1)?.seq ?? lastSeen
+  })
+
+  it('only gets events after cursor', async () => {
+    const count = 20
+    totalEvts += count
+    const outbox = new Outbox(sequencer)
+    const createPromise = createLabels(count)
+    const [evts] = await Promise.all([
+      readFromGenerator(
+        outbox.events(lastSeen),
+        caughtUp(outbox),
+        createPromise,
+      ),
+      createPromise,
+    ])
+
+    // +1 because we send the lastSeen date as well
+    expect(evts.length).toBe(count)
+
+    const fromDb = await loadFromDb(lastSeen)
+    expect(evts.map(evtToDbRow)).toEqual(fromDb)
+
+    lastSeen = evts.at(-1)?.seq ?? lastSeen
+  })
+
+  it('buffers events that are not being read', async () => {
+    const count = 20
+    totalEvts += count
+    const outbox = new Outbox(sequencer)
+    const createPromise = createLabels(count)
+    const gen = outbox.events(lastSeen)
+    // read enough to start streaming then wait so that the rest go into the buffer,
+    // then stream out from buffer
+    const [firstPart] = await Promise.all([
+      readFromGenerator(gen, caughtUp(outbox), createPromise, 5),
+      createPromise,
+    ])
+    const secondPart = await readFromGenerator(
+      gen,
+      caughtUp(outbox),
+      createPromise,
+    )
+    const evts = [...firstPart, ...secondPart]
+    expect(evts.length).toBe(count)
+
+    const fromDb = await loadFromDb(lastSeen)
+    expect(evts.map(evtToDbRow)).toEqual(fromDb)
+
+    lastSeen = evts.at(-1)?.seq ?? lastSeen
+  })
+
+  it('errors when buffer is overloaded', async () => {
+    const count = 20
+    totalEvts += count
+    const outbox = new Outbox(sequencer, { maxBufferSize: 5 })
+    const gen = outbox.events(lastSeen)
+    const createPromise = createLabels(count)
+    // read enough to start streaming then wait to stream rest until buffer is overloaded
+    const overloadBuffer = async () => {
+      await Promise.all([
+        readFromGenerator(gen, caughtUp(outbox), createPromise, 5),
+        createPromise,
+      ])
+      await wait(500)
+      await readFromGenerator(gen, caughtUp(outbox), createPromise)
+    }
+    await expect(overloadBuffer).rejects.toThrow('Stream consumer too slow')
+
+    await createPromise
+
+    const fromDb = await loadFromDb(lastSeen)
+    lastSeen = fromDb.at(-1)?.id ?? lastSeen
+  })
+
+  it('handles many open connections', async () => {
+    const count = 20
+    const outboxes: Outbox[] = []
+    for (let i = 0; i < 50; i++) {
+      outboxes.push(new Outbox(sequencer))
+    }
+    const createPromise = createLabels(count)
+    const readOutboxes = Promise.all(
+      outboxes.map((o) =>
+        readFromGenerator(o.events(lastSeen), caughtUp(o), createPromise),
+      ),
+    )
+    const [results] = await Promise.all([readOutboxes, createPromise])
+    const fromDb = await loadFromDb(lastSeen)
+    for (const result of results) {
+      expect(result.length).toBe(count)
+      expect(result.map(evtToDbRow)).toEqual(fromDb)
+    }
+    lastSeen = results[0].at(-1)?.seq ?? lastSeen
+  })
+})

package/tests/server.test.ts

@@ -55,6 +55,8 @@ describe('server', () => {
   })
 
   it('healthcheck fails when database is unavailable.', async () => {
+    // destory sequencer to release connection that would prevent the db from closing
+    await ozone.ctx.sequencer.destroy()
     await ozone.ctx.db.close()
     let error: AxiosError
     try {